Handle changing of node status

Author: knizhnik

arbiter.c

@@ -295,7 +295,7 @@ static int MtmConnectSocket(char const* host, int port, int max_attempts)
 				goto Retry;
 			}
 
-			/* Some node cnosidered that I am dead, so switch to recovery mode */
+			/* Some node considered that I am dead, so switch to recovery mode */
 			if (BIT_CHECK(msg.disabledNodeMask, MtmNodeId-1)) { 
 				elog(WARNING, "Node is switched to recovery mode");
 				ds->status = MTM_RECOVERY;

bkb.c

@@ -98,7 +98,7 @@ static void findMaximumIndependentSet(NodeList* cur, NodeList* result, nodemask_
 	}
 }
 
-nodemask_t MtmFindMaxClique(nodemask_t* graph, int n_nodes)
+nodemask_t MtmFindMaxClique(nodemask_t* graph, int n_nodes, int* clique_size)
 {
 	NodeList tmp;
 	NodeList result;
@@ -116,5 +116,6 @@ nodemask_t MtmFindMaxClique(nodemask_t* graph, int n_nodes)
 	for (i = 0; i < result.size; i++) { 
 		BIT_SET(mask, result.nodes[i]);
 	}
+	*clique_size = result.size;
 	return mask;
 }

bkb.h

@@ -11,6 +11,6 @@ typedef uint64_t nodemask_t;
 #define BIT_CLEAR(mask, bit) (mask &= ~((nodemask_t)1 << (bit)))
 #define BIT_SET(mask, bit)   (mask |= ((nodemask_t)1 << (bit)))
 
-extern nodemask_t MtmFindMaxClique(nodemask_t* matrix, int n_modes);
+extern nodemask_t MtmFindMaxClique(nodemask_t* matrix, int n_modes, int* clique_size);
 
 #endif

multimaster.c

@@ -1084,13 +1084,20 @@ _PG_fini(void)
  */
 
 
-static void MtmSwitchFromRecoveryToNormalMode()
+void MtmSwitchToNormalMode()
 {
 	dtm->status = MTM_ONLINE;
 	elog(WARNING, "Switch to normal mode");
 	/* ??? Something else to do here? */
 }
 
+void MtmSwitchToRecoveryMode()
+{
+	dtm->status = MTM_RECOVERY;
+	/* ??? Something else to do here? */
+	elog(ERROR, "Switch to normal mode");
+}
+
 
 void MtmJoinTransaction(GlobalTransactionId* gtid, csn_t globalSnapshot)
 {
@@ -1110,7 +1117,7 @@ void MtmJoinTransaction(GlobalTransactionId* gtid, csn_t globalSnapshot)
 		Assert(dtm->status == MTM_RECOVERY);
 	} else if (dtm->status == MTM_RECOVERY) { 
 		/* When recovery is completed we get normal transaction ID and switch to normal mode */
-		MtmSwitchFromRecoveryToNormalMode();
+		MtmSwitchToNormalMode();
 	}
 	dtmTx.gtid = *gtid;
 	dtmTx.xid = GetCurrentTransactionId();
@@ -1646,41 +1653,51 @@ MtmDetectGlobalDeadLock(PGPROC* proc)
 static void 
 MtmBuildConnectivityMatrix(nodemask_t* matrix)
 {
-	int i;
-	for (i = 0; i < MtmNodes; i++) { 
+	int i, j, n = MtmNodes;
+	for (i = 0; i < n; i++) { 
 		if (i+1 != MtmNodeId) { 
 			void* data = PaxosGet(psprintf("node-mask-%d", i+1), NULL, NULL);
 			matrix[i] = *(nodemask_t*)data;
 		} else { 
 			matrix[i] = dtm->connectivityMask;
 		}
 	}
+	/* make matrix symetric: required for Bron–Kerbosch algorithm */
+	for (i = 0; i < n; i++) { 
+		for (j = 0; j < i; j++) { 
+			matrix[i] |= ((matrix[j] >> i) & 1) << j;
+		}
+	}
 }	
 
 
 void MtmUpdateClusterStatus(void)
 {
-	nodemask_t mask, clique, disconnectedMask;
+	nodemask_t mask, clique;
 	nodemask_t matrix[MAX_NODES];
+	int clique_size;
 	int i;
 
 	MtmBuildConnectivityMatrix(matrix);
 
-	clique = MtmFindMaxClique(matrix, MtmNodes);
-	disconnectedMask = ~clique & (((nodemask_t)1 << MtmNodes)-1);
-	MtmLock(LW_EXCLUSIVE);
-	mask = disconnectedMask & ~dtm->disabledNodeMask;
-	for (i = 0; mask != 0; i++, mask >>= 1) {
-		if (mask & 1) { 
-			dtm->nNodes -= 1;
-			BIT_SET(dtm->disabledNodeMask, i);
+	clique = MtmFindMaxClique(matrix, MtmNodes, &clique_size);
+	if (clique_size >= MtmNodes/2+1) { /* have quorum */
+		MtmLock(LW_EXCLUSIVE);
+		mask = ~clique & (((nodemask_t)1 << MtmNodes)-1) & ~dtm->disabledNodeMask;
+		for (i = 0; mask != 0; i++, mask >>= 1) {
+			if (mask & 1) { 
+				dtm->nNodes -= 1;
+				BIT_SET(dtm->disabledNodeMask, i);
+			}
 		}
+		MtmUnlock();
+		if (BIT_CHECK(dtm->disabledNodeMask, MtmNodeId-1)) { 
+            /* I was excluded from cluster:( */
+			MtmSwitchToRecoveryMode();
+		}
+	} else { 
+		elog(WARNING, "Clique %lx has no quorum", clique);
 	}
-	if (dtm->disabledNodeMask != disconnectedMask) { 
-		dtm->disabledNodeMask |= disconnectedMask;
-		PaxosSet(psprintf("node-mask-%d", MtmNodeId), &dtm->disabledNodeMask, sizeof dtm->disabledNodeMask);
-	}
-	MtmUnlock();
 }
 
 void MtmOnNodeDisconnect(int nodeId)

multimaster.h

@@ -148,4 +148,6 @@ extern timestamp_t MtmGetCurrentTime(void);
 extern void  MtmSleep(timestamp_t interval);
 extern bool  MtmIsRecoveredNode(int nodeId);
 extern void  MtmUpdateClusterStatus(void);
+extern void MtmSwitchToNormalMode(void);
+extern void MtmSwitchToRecoveryMode(void);
 #endif