Skip to content

WIP Fix mqbc: Cluster FSM must heal before starting Partition FSMs #951

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

WIP Fix mqbc: Cluster FSM must heal before starting Partition FSMs #951

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c06658d
mqbc::PartitionStateTable: Unsupported primary downgrade
kaikulimu Nov 18, 2025
ca92cbd
mqbc::StorageMgr: 'dispatchEventToPartition' supports execute in-place
kaikulimu Nov 19, 2025
07f6b5f
mqbc: Update partition info before invoking Partition FSMs
kaikulimu Nov 19, 2025
eea82c5
TODO rm log
kaikulimu Sep 30, 2025
c6f9978
TMP queueKeyInfoMap comment
kaikulimu Sep 30, 2025
b5df771
TMP
kaikulimu Nov 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions src/groups/mqb/mqbblp/mqbblp_storagemanager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
// limitations under the License.

// mqbblp_storagemanager.cpp -*-C++-*-
#include <ball_log.h>
#include <mqbblp_storagemanager.h>

#include <mqbscm_version.h>
Expand Down Expand Up @@ -606,12 +607,22 @@ void StorageManager::clearPrimaryForPartitionDispatched(
mqbs::FileStore* fs = d_fileStores[partitionId].get();
PartitionInfo& pinfo = d_partitionInfoVec[partitionId];

if (primary != pinfo.primary()) {
BALL_LOG_WARN << d_clusterData_p->identity().description()
<< " Partition [" << partitionId
<< "]: Failed to clear primary as specified primary: "
<< (primary ? primary->nodeDescription() : "**null**")
<< " is different from current perceived primary: "
<< (pinfo.primary() ? pinfo.primary()->nodeDescription()
: "** null **");
return; // RETURN
}

mqbc::StorageUtil::clearPrimaryForPartition(
fs,
&pinfo,
d_clusterData_p->identity().description(),
partitionId,
primary);
partitionId);
}

void StorageManager::processStorageEventDispatched(
Expand Down
4 changes: 2 additions & 2 deletions src/groups/mqb/mqbc/mqbc_clusterstate.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -701,7 +701,7 @@ class ClusterState {
// ACCESSORS
/// Return the value of the corresponding member of this object.
const mqbi::Cluster* cluster() const;
const PartitionsInfo& partitionsInfo() const;
const PartitionsInfo& partitionsInfo() const; // TODO why diff name?!
const DomainStates& domainStates() const;
const QueueKeys& queueKeys() const;
const ObserversSet& observers() const;
Expand Down Expand Up @@ -737,7 +737,7 @@ class ClusterState {
bool hasActivePrimary(int partitionId) const;

/// Return a reference to the partitions info.
const PartitionsInfo& partitions() const;
const PartitionsInfo& partitions() const; // TODO why diff name?!

/// Return a reference to the PartitionInfo corresponding to the
/// specified `partitionId`. This method is the same as
Expand Down
145 changes: 50 additions & 95 deletions src/groups/mqb/mqbc/mqbc_partitionstatetable.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,10 +211,6 @@ class PartitionStateTableActions {

virtual void do_storeReplicaSeq(const ARGS& args) = 0;

virtual void do_storePartitionInfo(const ARGS& args) = 0;

virtual void do_clearPartitionInfo(const ARGS& args) = 0;

virtual void do_replicaStateRequest(const ARGS& args) = 0;

virtual void do_replicaStateResponse(const ARGS& args) = 0;
Expand Down Expand Up @@ -296,11 +292,10 @@ class PartitionStateTableActions {
const ARGS& args);

void
do_startWatchDog_storePartitionInfo_openRecoveryFileSet_storeSelfSeq_replicaStateRequest_checkQuorumSeq(
do_startWatchDog_openRecoveryFileSet_storeSelfSeq_replicaStateRequest_checkQuorumSeq(
const ARGS& args);

void
do_startWatchDog_storePartitionInfo_openRecoveryFileSet_storeSelfSeq_primaryStateRequest(
void do_startWatchDog_openRecoveryFileSet_storeSelfSeq_primaryStateRequest(
const ARGS& args);

void
Expand All @@ -310,16 +305,12 @@ class PartitionStateTableActions {

void do_storePrimarySeq_replicaStateResponse(const ARGS& args);

void do_cleanupMetadata_clearPartitionInfo_reapplyEvent(const ARGS& args);

void do_cleanupMetadata_clearPartitionInfo_stopWatchDog_reapplyEvent(
const ARGS& args);
void do_cleanupMetadata_reapplyEvent(const ARGS& args);

void do_cleanupMetadata_clearPartitionInfo(const ARGS& args);
void do_cleanupMetadata_stopWatchDog_reapplyEvent(const ARGS& args);

void
do_cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog(
const ARGS& args);
do_cleanupMetadata_closeRecoveryFileSet_stopWatchDog(const ARGS& args);

void do_cleanupMetadata_closeRecoveryFileSet_reapplyDetectSelfPrimary(
const ARGS& args);
Expand Down Expand Up @@ -351,8 +342,7 @@ class PartitionStateTableActions {
void do_cleanupMetadata_reapplyDetectSelfReplica(const ARGS& args);

void
do_resetReceiveDataCtx_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog(
const ARGS& args);
do_resetReceiveDataCtx_closeRecoveryFileSet_stopWatchDog(const ARGS& args);

void do_cleanupMetadata_closeRecoveryFileSet_reapplyDetectSelfReplica(
const ARGS& args);
Expand Down Expand Up @@ -434,17 +424,17 @@ class PartitionStateTable
PST_CFG(
UNKNOWN,
DETECT_SELF_PRIMARY,
startWatchDog_storePartitionInfo_openRecoveryFileSet_storeSelfSeq_replicaStateRequest_checkQuorumSeq,
startWatchDog_openRecoveryFileSet_storeSelfSeq_replicaStateRequest_checkQuorumSeq,
PRIMARY_HEALING_STG1);
PST_CFG(
UNKNOWN,
DETECT_SELF_REPLICA,
startWatchDog_storePartitionInfo_openRecoveryFileSet_storeSelfSeq_primaryStateRequest,
startWatchDog_openRecoveryFileSet_storeSelfSeq_primaryStateRequest,
REPLICA_HEALING);
PST_CFG(UNKNOWN, STOP_NODE, none, STOPPED);
PST_CFG(PRIMARY_HEALING_STG1,
DETECT_SELF_REPLICA,
cleanupMetadata_clearPartitionInfo_stopWatchDog_reapplyEvent,
unsupportedPrimaryDowngrade,
UNKNOWN);
PST_CFG(PRIMARY_HEALING_STG1,
REPLICA_STATE_RQST,
Expand Down Expand Up @@ -475,23 +465,21 @@ class PartitionStateTable
REPLICA_HIGHEST_SEQ,
setExpectedDataChunkRange_replicaDataRequestPull,
PRIMARY_HEALING_STG2);
PST_CFG(
PRIMARY_HEALING_STG1,
RST_UNKNOWN,
cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog,
UNKNOWN);
PST_CFG(
PRIMARY_HEALING_STG1,
STOP_NODE,
cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog,
STOPPED);
PST_CFG(PRIMARY_HEALING_STG1,
RST_UNKNOWN,
cleanupMetadata_closeRecoveryFileSet_stopWatchDog,
UNKNOWN);
PST_CFG(PRIMARY_HEALING_STG1,
STOP_NODE,
cleanupMetadata_closeRecoveryFileSet_stopWatchDog,
STOPPED);
PST_CFG(PRIMARY_HEALING_STG1,
WATCH_DOG,
cleanupMetadata_closeRecoveryFileSet_reapplyDetectSelfPrimary,
UNKNOWN);
PST_CFG(PRIMARY_HEALING_STG2,
DETECT_SELF_REPLICA,
cleanupMetadata_clearPartitionInfo_stopWatchDog_reapplyEvent,
unsupportedPrimaryDowngrade,
UNKNOWN);
PST_CFG(PRIMARY_HEALING_STG2,
FAIL_REPLICA_DATA_RSPN_PULL,
Expand Down Expand Up @@ -533,27 +521,25 @@ class PartitionStateTable
QUORUM_REPLICA_DATA_RSPN,
stopWatchDog_transitionToActivePrimary,
PRIMARY_HEALED);
PST_CFG(
PRIMARY_HEALING_STG2,
RST_UNKNOWN,
cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog,
UNKNOWN);
PST_CFG(PRIMARY_HEALING_STG2,
RST_UNKNOWN,
cleanupMetadata_closeRecoveryFileSet_stopWatchDog,
UNKNOWN);
PST_CFG(PRIMARY_HEALING_STG2,
WATCH_DOG,
cleanupMetadata_closeRecoveryFileSet_reapplyDetectSelfPrimary,
UNKNOWN);
PST_CFG(
PRIMARY_HEALING_STG2,
STOP_NODE,
cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog,
STOPPED);
PST_CFG(PRIMARY_HEALING_STG2,
STOP_NODE,
cleanupMetadata_closeRecoveryFileSet_stopWatchDog,
STOPPED);
PST_CFG(REPLICA_HEALING,
DETECT_SELF_PRIMARY,
cleanupMetadata_clearPartitionInfo_stopWatchDog_reapplyEvent,
cleanupMetadata_stopWatchDog_reapplyEvent,
UNKNOWN);
PST_CFG(REPLICA_HEALING,
DETECT_SELF_REPLICA,
cleanupMetadata_clearPartitionInfo_stopWatchDog_reapplyEvent,
cleanupMetadata_stopWatchDog_reapplyEvent,
UNKNOWN);
PST_CFG(REPLICA_HEALING,
REPLICA_STATE_RQST,
Expand Down Expand Up @@ -608,27 +594,25 @@ class PartitionStateTable
failureReplicaDataResponsePush_cleanupMetadata_closeRecoveryFileSet_stopWatchDog_reapplyDetectSelfReplica,
UNKNOWN);
PST_CFG(REPLICA_HEALING, LIVE_DATA, bufferLiveData, REPLICA_HEALING);
PST_CFG(
REPLICA_HEALING,
RST_UNKNOWN,
cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog,
UNKNOWN);
PST_CFG(REPLICA_HEALING,
RST_UNKNOWN,
cleanupMetadata_closeRecoveryFileSet_stopWatchDog,
UNKNOWN);
PST_CFG(REPLICA_HEALING,
WATCH_DOG,
cleanupMetadata_closeRecoveryFileSet_reapplyDetectSelfReplica,
UNKNOWN);
PST_CFG(
REPLICA_HEALING,
STOP_NODE,
cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog,
STOPPED);
PST_CFG(REPLICA_HEALING,
STOP_NODE,
cleanupMetadata_closeRecoveryFileSet_stopWatchDog,
STOPPED);
PST_CFG(REPLICA_HEALED,
DETECT_SELF_PRIMARY,
cleanupMetadata_clearPartitionInfo_reapplyEvent,
cleanupMetadata_reapplyEvent,
UNKNOWN);
PST_CFG(REPLICA_HEALED,
DETECT_SELF_REPLICA,
cleanupMetadata_clearPartitionInfo_reapplyEvent,
cleanupMetadata_reapplyEvent,
UNKNOWN);
PST_CFG(REPLICA_HEALED,
REPLICA_STATE_RQST,
Expand All @@ -643,14 +627,8 @@ class PartitionStateTable
ISSUE_LIVESTREAM,
cleanupMetadata_reapplyDetectSelfReplica,
UNKNOWN);
PST_CFG(REPLICA_HEALED,
RST_UNKNOWN,
cleanupMetadata_clearPartitionInfo,
UNKNOWN);
PST_CFG(REPLICA_HEALED,
STOP_NODE,
cleanupMetadata_clearPartitionInfo,
STOPPED);
PST_CFG(REPLICA_HEALED, RST_UNKNOWN, cleanupMetadata, UNKNOWN);
PST_CFG(REPLICA_HEALED, STOP_NODE, cleanupMetadata, STOPPED);
PST_CFG(PRIMARY_HEALED,
DETECT_SELF_REPLICA,
unsupportedPrimaryDowngrade,
Expand All @@ -665,14 +643,8 @@ class PartitionStateTable
PRIMARY_STATE_RQST,
storeSelfSeq_storeReplicaSeq_primaryStateResponse_replicaDataRequestPush_replicaDataRequestDrop_startSendDataChunks,
PRIMARY_HEALED);
PST_CFG(PRIMARY_HEALED,
RST_UNKNOWN,
cleanupMetadata_clearPartitionInfo,
UNKNOWN);
PST_CFG(PRIMARY_HEALED,
STOP_NODE,
cleanupMetadata_clearPartitionInfo,
STOPPED);
PST_CFG(PRIMARY_HEALED, RST_UNKNOWN, cleanupMetadata, UNKNOWN);
PST_CFG(PRIMARY_HEALED, STOP_NODE, cleanupMetadata, STOPPED);

#undef PST_CFG
}
Expand Down Expand Up @@ -718,11 +690,10 @@ void PartitionStateTableActions<ARGS>::

template <typename ARGS>
void PartitionStateTableActions<ARGS>::
do_startWatchDog_storePartitionInfo_openRecoveryFileSet_storeSelfSeq_replicaStateRequest_checkQuorumSeq(
do_startWatchDog_openRecoveryFileSet_storeSelfSeq_replicaStateRequest_checkQuorumSeq(
const ARGS& args)
{
do_startWatchDog(args);
do_storePartitionInfo(args);
do_openRecoveryFileSet(args);
do_storeSelfSeq(args);
do_replicaStateRequest(args);
Expand All @@ -731,11 +702,10 @@ void PartitionStateTableActions<ARGS>::

template <typename ARGS>
void PartitionStateTableActions<ARGS>::
do_startWatchDog_storePartitionInfo_openRecoveryFileSet_storeSelfSeq_primaryStateRequest(
do_startWatchDog_openRecoveryFileSet_storeSelfSeq_primaryStateRequest(
const ARGS& args)
{
do_startWatchDog(args);
do_storePartitionInfo(args);
do_openRecoveryFileSet(args);
do_storeSelfSeq(args);
do_primaryStateRequest(args);
Expand Down Expand Up @@ -767,40 +737,27 @@ void PartitionStateTableActions<ARGS>::do_storePrimarySeq_replicaStateResponse(
}

template <typename ARGS>
void PartitionStateTableActions<
ARGS>::do_cleanupMetadata_clearPartitionInfo_reapplyEvent(const ARGS& args)
void PartitionStateTableActions<ARGS>::do_cleanupMetadata_reapplyEvent(
const ARGS& args)
{
do_cleanupMetadata(args);
do_clearPartitionInfo(args);
do_reapplyEvent(args);
}

template <typename ARGS>
void PartitionStateTableActions<ARGS>::
do_cleanupMetadata_clearPartitionInfo_stopWatchDog_reapplyEvent(
const ARGS& args)
void PartitionStateTableActions<
ARGS>::do_cleanupMetadata_stopWatchDog_reapplyEvent(const ARGS& args)
{
do_cleanupMetadata(args);
do_clearPartitionInfo(args);
do_stopWatchDog(args);
do_reapplyEvent(args);
}

template <typename ARGS>
void PartitionStateTableActions<ARGS>::do_cleanupMetadata_clearPartitionInfo(
const ARGS& args)
{
do_cleanupMetadata(args);
do_clearPartitionInfo(args);
}

template <typename ARGS>
void PartitionStateTableActions<ARGS>::
do_cleanupMetadata_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog(
const ARGS& args)
do_cleanupMetadata_closeRecoveryFileSet_stopWatchDog(const ARGS& args)
{
do_cleanupMetadata(args);
do_clearPartitionInfo(args);
do_closeRecoveryFileSet(args);
do_stopWatchDog(args);
}
Expand Down Expand Up @@ -914,11 +871,9 @@ void PartitionStateTableActions<

template <typename ARGS>
void PartitionStateTableActions<ARGS>::
do_resetReceiveDataCtx_clearPartitionInfo_closeRecoveryFileSet_stopWatchDog(
const ARGS& args)
do_resetReceiveDataCtx_closeRecoveryFileSet_stopWatchDog(const ARGS& args)
{
do_resetReceiveDataCtx(args);
do_clearPartitionInfo(args);
do_closeRecoveryFileSet(args);
do_stopWatchDog(args);
}
Expand Down
Loading
Loading