@@ -635,9 +635,11 @@ MtmBeginTransaction(MtmCurrentTrans* x)
635
635
x -> isDistributed = MtmIsUserTransaction ();
636
636
x -> isPrepared = false;
637
637
x -> isTransactionBlock = IsTransactionBlock ();
638
- /* Application name can be cahnged usnig PGAPPNAME environment variable */
638
+ /* Application name can be changed usnig PGAPPNAME environment variable */
639
639
if (!IsBackgroundWorker && x -> isDistributed && Mtm -> status != MTM_ONLINE && strcmp (application_name , MULTIMASTER_ADMIN ) != 0 ) {
640
- /* reject all user's transactions at offline cluster */
640
+ /* Reject all user's transactions at offline cluster.
641
+ * Allow execution of transaction by bg-workers to make it possible to perform recovery.
642
+ */
641
643
MtmUnlock ();
642
644
elog (ERROR , "Multimaster node is not online: current status %s" , MtmNodeStatusMnem [Mtm -> status ]);
643
645
}
@@ -673,14 +675,17 @@ MtmPrePrepareTransaction(MtmCurrentTrans* x)
673
675
if (Mtm -> disabledNodeMask != 0 ) {
674
676
MtmRefreshClusterStatus (true);
675
677
if (!IsBackgroundWorker && Mtm -> status != MTM_ONLINE ) {
676
- elog (ERROR , "Abort current transaction because this cluster node is not online" );
678
+ /* Do not take in accoutn bg-workers which are performing recovery */
679
+ elog (ERROR , "Abort current transaction because this cluster node is in %s status" , MtmNodeStatusMnem [Mtm -> status ]);
677
680
}
678
681
}
679
682
680
683
MtmLock (LW_EXCLUSIVE );
681
684
682
685
/*
683
- * Check if there is global multimaster lock preventing new transaction from commit to make a chance to wal-senders to catch-up
686
+ * Check if there is global multimaster lock preventing new transaction from commit to make a chance to wal-senders to catch-up.
687
+ * Only "own" transactions are blacked. Transactions replicated from other nodes (including recovered transaction) should be proceeded
688
+ * and should not cause cluster status change.
684
689
*/
685
690
if (!x -> isReplicated ) {
686
691
MtmCheckClusterLock ();
@@ -716,7 +721,8 @@ MtmPrePrepareTransaction(MtmCurrentTrans* x)
716
721
}
717
722
MtmTransactionListAppend (ts );
718
723
MtmAddSubtransactions (ts , subxids , ts -> nSubxids );
719
- MTM_TRACE ("%d: MtmPrePrepareTransaction prepare commit of %d CSN=%ld\n" , MyProcPid , x -> xid , ts -> csn );
724
+ MTM_TRACE ("%d: MtmPrePrepareTransaction prepare commit of %d (gtid.xid=%d, gtid.node=%d, CSN=%ld)\n" ,
725
+ MyProcPid , x -> xid , ts -> gtid .xid , ts -> gtid .node , ts -> csn );
720
726
MtmUnlock ();
721
727
722
728
}
@@ -842,14 +848,6 @@ void MtmSendNotificationMessage(MtmTransState* ts, MtmMessageCode cmd)
842
848
}
843
849
}
844
850
845
- void MtmRecoveryCompleted (void )
846
- {
847
- elog (WARNING , "Recovery of node %d is completed" , MtmNodeId );
848
- Mtm -> recoverySlot = 0 ;
849
- BIT_CLEAR (Mtm -> disabledNodeMask , MtmNodeId - 1 );
850
- MtmSwitchClusterMode (MTM_ONLINE );
851
- }
852
-
853
851
void MtmJoinTransaction (GlobalTransactionId * gtid , csn_t globalSnapshot )
854
852
{
855
853
MtmLock (LW_EXCLUSIVE );
@@ -933,6 +931,18 @@ csn_t MtmGetTransactionCSN(TransactionId xid)
933
931
* -------------------------------------------
934
932
*/
935
933
934
+ void MtmRecoveryCompleted (void )
935
+ {
936
+ elog (WARNING , "Recovery of node %d is completed" , MtmNodeId );
937
+ MtmLock (LW_EXCLUSIVE );
938
+ Mtm -> recoverySlot = 0 ;
939
+ BIT_CLEAR (Mtm -> disabledNodeMask , MtmNodeId - 1 );
940
+ /* Mode will be changed to online once all locagical reciever are connected */
941
+ MtmSwitchClusterMode (MTM_CONNECTED );
942
+ MtmUnlock ();
943
+ }
944
+
945
+
936
946
937
947
/**
938
948
* Check state of replication slots. If some of them are too much lag behind wal, then drop this slots to avoid
@@ -993,10 +1003,10 @@ bool MtmIsRecoveredNode(int nodeId)
993
1003
bool MtmRecoveryCaughtUp (int nodeId , XLogRecPtr slotLSN )
994
1004
{
995
1005
bool caughtUp = false;
1006
+ MtmLock (LW_EXCLUSIVE );
996
1007
if (MtmIsRecoveredNode (nodeId )) {
997
1008
XLogRecPtr walLSN = GetXLogInsertRecPtr ();
998
- MtmLock (LW_EXCLUSIVE );
999
- if (slotLSN == walLSN ) {
1009
+ if (slotLSN == walLSN && Mtm -> nActiveTransactions == 0 ) {
1000
1010
if (BIT_CHECK (Mtm -> nodeLockerMask , nodeId - 1 )) {
1001
1011
elog (WARNING ,"Node %d is caught-up" , nodeId );
1002
1012
BIT_CLEAR (Mtm -> walSenderLockerMask , MyWalSnd - WalSndCtl -> walsnds );
@@ -1018,18 +1028,17 @@ bool MtmRecoveryCaughtUp(int nodeId, XLogRecPtr slotLSN)
1018
1028
* We have to maintain two bitmasks: one is marking wal sender, another - correspondent nodes.
1019
1029
* Is there some better way to establish mapping between nodes ad WAL-seconder?
1020
1030
*/
1021
- elog (WARNING ,"Node %d is almost caught-up: lock cluster" , nodeId );
1031
+ elog (WARNING ,"Node %d is almost caught-up: slot position %lx, WAL position %lx, active transactions %d" ,
1032
+ nodeId , slotLSN , walLSN , Mtm -> nActiveTransactions );
1022
1033
Assert (MyWalSnd != NULL ); /* This function is called by WAL-sender, so it should not be NULL */
1023
1034
BIT_SET (Mtm -> nodeLockerMask , nodeId - 1 );
1024
1035
BIT_SET (Mtm -> walSenderLockerMask , MyWalSnd - WalSndCtl -> walsnds );
1025
1036
Mtm -> nLockers += 1 ;
1026
1037
} else {
1027
1038
MTM_INFO ("Continue recovery of node %d, slot position %lx, WAL position %lx, WAL sender position %lx, lockers %d, active transactions %d\n" , nodeId , slotLSN , walLSN , MyWalSnd -> sentPtr , Mtm -> nLockers , Mtm -> nActiveTransactions );
1028
1039
}
1029
- MtmUnlock ();
1030
- } else {
1031
- MTM_INFO ("Node %d is not in recovery mode\n" , nodeId );
1032
1040
}
1041
+ MtmUnlock ();
1033
1042
return caughtUp ;
1034
1043
}
1035
1044
@@ -1044,7 +1053,7 @@ void MtmSwitchClusterMode(MtmNodeStatus mode)
1044
1053
/*
1045
1054
* If there are recovering nodes which are catching-up WAL, check the status and prevent new transaction from commit to give
1046
1055
* WAL-sender a chance to catch-up WAL, completely synchronize replica and switch it to normal mode.
1047
- * This function is called at transaction start with multimaster lock set
1056
+ * This function is called before transaction prepare with multimaster lock set.
1048
1057
*/
1049
1058
static void
1050
1059
MtmCheckClusterLock ()
@@ -1071,8 +1080,8 @@ MtmCheckClusterLock()
1071
1080
}
1072
1081
}
1073
1082
if (mask != 0 ) {
1074
- /* some "almost catch-up" wal-senders are still working */
1075
- /* Do not start new transactions until them complete */
1083
+ /* some "almost catch-up" wal-senders are still working. */
1084
+ /* Do not start new transactions until them are completed. */
1076
1085
MtmUnlock ();
1077
1086
MtmSleep (delay );
1078
1087
if (delay * 2 <= MAX_WAIT_TIMEOUT ) {
@@ -1215,6 +1224,7 @@ void MtmOnNodeDisconnect(int nodeId)
1215
1224
void MtmOnNodeConnect (int nodeId )
1216
1225
{
1217
1226
BIT_CLEAR (Mtm -> connectivityMask , nodeId - 1 );
1227
+ elog (NOTICE , "Reconnect node %d" , nodeId );
1218
1228
RaftableSet (psprintf ("node-mask-%d" , MtmNodeId ), & Mtm -> connectivityMask , sizeof Mtm -> connectivityMask , false);
1219
1229
}
1220
1230
@@ -1645,19 +1655,23 @@ _PG_fini(void)
1645
1655
}
1646
1656
1647
1657
1648
-
1658
+ /*
1659
+ * This functions is called by pglogical receiver main function when receiver background worker is started.
1660
+ * We switch to ONLINE mode when all receviers are connected.
1661
+ * As far as background worker can be restarted multiple times, use node bitmask.
1662
+ */
1649
1663
void MtmReceiverStarted (int nodeId )
1650
1664
{
1651
- SpinLockAcquire ( & Mtm -> spinlock );
1665
+ MtmLock ( LW_EXCLUSIVE );
1652
1666
if (!BIT_CHECK (Mtm -> pglogicalNodeMask , nodeId - 1 )) {
1653
1667
BIT_SET (Mtm -> pglogicalNodeMask , nodeId - 1 );
1654
1668
if (++ Mtm -> nReceivers == Mtm -> nNodes - 1 ) {
1655
1669
if (Mtm -> status == MTM_CONNECTED ) {
1656
1670
MtmSwitchClusterMode (MTM_ONLINE );
1657
1671
}
1658
1672
}
1659
- }
1660
- SpinLockRelease ( & Mtm -> spinlock );
1673
+ }
1674
+ MtmUnlock ();
1661
1675
}
1662
1676
1663
1677
/*
0 commit comments