Check for heartbeat in raftable_set

Author: knizhnik

contrib/mmts/arbiter.c

@@ -112,8 +112,6 @@ static int         busy_socket;
 static void MtmTransSender(Datum arg);
 static void MtmTransReceiver(Datum arg);
 static void MtmSendHeartbeat(void);
-static void MtmCheckHeartbeat(void);
-
 
 
 static char const* const messageText[] = 
@@ -356,7 +354,7 @@ static void MtmSendHeartbeat()
 
 }
 
-static void MtmCheckHeartbeat()
+void MtmCheckHeartbeat()
 {
 	if (send_heartbeat) {
 		send_heartbeat = false;

contrib/mmts/multimaster.c

@@ -796,8 +796,8 @@ void MtmWatchdog(void)
 			if (Mtm->nodes[i].lastHeartbeat != 0
 				&& now > Mtm->nodes[i].lastHeartbeat + MSEC_TO_USEC(MtmHeartbeatRecvTimeout)) 
 			{ 
-				elog(WARNING, "Disable node %d because last heartbeat was received %d msec ago (%ld)", 
-					 i+1, (int)USEC_TO_MSEC(now - Mtm->nodes[i].lastHeartbeat), USEC_TO_MSEC(now));
+				elog(WARNING, "Heartbeat was received from node %d during %d msec", 
+					 i+1, (int)USEC_TO_MSEC(now - Mtm->nodes[i].lastHeartbeat));
 				MtmOnNodeDisconnect(i+1);				
 			}
 		}
@@ -840,24 +840,20 @@ MtmPostPrepareTransaction(MtmCurrentTrans* x)
 
 		timestamp_t start = MtmGetSystemTime();	
 		/* wait votes from all nodes */
-		while (!ts->votingCompleted && start + transTimeout >= MtmGetSystemTime()) 
+		while (!ts->votingCompleted && ts->status != TRANSACTION_STATUS_ABORTED && start + transTimeout >= MtmGetSystemTime()) 
 		{
 			MtmUnlock();
-			MtmWatchdog();
-			if (ts->status == TRANSACTION_STATUS_ABORTED) {
-				elog(WARNING, "Transaction %d(%s) is aborted by watchdog", x->xid, x->gid);				
-				x->status = TRANSACTION_STATUS_ABORTED;
-				return;
-			}
 			result = WaitLatch(&MyProc->procLatch, WL_LATCH_SET|WL_TIMEOUT, MtmHeartbeatRecvTimeout);
 			if (result & WL_LATCH_SET) { 
 				ResetLatch(&MyProc->procLatch);			
 			} 
 			MtmLock(LW_SHARED);
 		}
-		if (!ts->votingCompleted) { 
-			MtmAbortTransaction(ts);
-			elog(WARNING, "Transaction is aborted because of %d msec timeout expiration, prepare time %d msec", (int)transTimeout, (int)USEC_TO_MSEC(ts->csn - x->snapshot));
+		if (!ts->votingCompleted) {  
+			if (ts->status != TRANSACTION_STATUS_ABORTED) { 
+				MtmAbortTransaction(ts);
+				elog(WARNING, "Transaction is aborted because of %d msec timeout expiration, prepare time %d msec", (int)transTimeout, (int)USEC_TO_MSEC(ts->csn - x->snapshot));
+			}
 		} else if (nConfigChanges != Mtm->nConfigChanges) {
 			MtmAbortTransaction(ts);
 			elog(WARNING, "Transaction is aborted because cluster configuration is changed during commit");
@@ -1436,7 +1432,7 @@ void MtmOnNodeDisconnect(int nodeId)
 	BIT_SET(Mtm->reconnectMask, nodeId-1);
 	MtmUnlock();
 
-	RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, true); /* false); -- TODO: raftable is hanged with nowait=true */
+	RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, false);
 
 	MtmSleep(MSEC_TO_USEC(MtmHeartbeatSendTimeout));
 
@@ -1470,7 +1466,7 @@ void MtmOnNodeConnect(int nodeId)
 	MtmUnlock();
 
 	MTM_LOG1("Reconnect node %d", nodeId);
-	RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, true); /* false); -- TODO: raftable is hanged with nowait=true */
+	RaftableSet(psprintf("node-mask-%d", MtmNodeId), &Mtm->connectivityMask, sizeof Mtm->connectivityMask, false); 
 }
 
 

contrib/mmts/multimaster.h

@@ -5,6 +5,7 @@
 #include "bgwpool.h"
 #include "bkb.h"
 
+#include "access/clog.h"
 #include "pglogical_output/hooks.h"
 
 #define  DEBUG_LEVEL 0
@@ -268,6 +269,8 @@ extern void  MtmHandleApplyError(void);
 extern void  MtmUpdateLsnMapping(int nodeId, XLogRecPtr endLsn);
 extern XLogRecPtr MtmGetFlushPosition(int nodeId);
 extern void MtmWatchdog(void);
+extern void MtmCheckHeartbeat(void);
+
 
 
 #endif

contrib/mmts/raftable.c

@@ -2,7 +2,7 @@
 #include "postgres.h"
 #include "raftable.h"
 #include "raftable_wrapper.h"
-
+#include "multimaster.h"
 
 /*
  * Raftable function proxies
@@ -18,8 +18,14 @@ void* RaftableGet(char const* key, size_t* size, RaftableTimestamp* ts, bool now
 
 void RaftableSet(char const* key, void const* value, size_t size, bool nowait)
 {
-	if (MtmUseRaftable) { 
-		raftable_set(key, value, size, nowait ? 0 : -1);
+	if (MtmUseRaftable) {
+		if (nowait) {
+			raftable_set(key, value, size, 0);
+		} else { 
+			while (!raftable_set(key, value, size, MtmHeartbeatSendTimeout)) { 
+				MtmCheckHeartbeat();
+			}
+		}
 	}		
 }