Move assertion about valid CSN from sender to receiver. It is okay when such tx will be filtered anyway

Author: kelvich

multimaster.c

@@ -1849,8 +1849,7 @@ csn_t MtmGetTransactionCSN(TransactionId xid)
 	csn_t csn;
 	MtmLock(LW_SHARED);
 	ts = (MtmTransState*)hash_search(MtmXid2State, &xid, HASH_FIND, NULL);
-	Assert(ts != NULL);
-	csn = ts->csn;
+	csn = ts ? ts->csn : INVALID_CSN;
 	MtmUnlock();
 	return csn;
 }

pglogical_apply.c

@@ -735,7 +735,14 @@ process_remote_commit(StringInfo in)
 		case PGLOGICAL_COMMIT_PREPARED:
 		{
 			Assert(!TransactionIdIsValid(MtmGetCurrentTransactionId()));
-			csn = pq_getmsgint64(in); 
+			csn = pq_getmsgint64(in);
+			/*
+			 * Since our recovery method allows undershoot of csn, we can receive
+			 * some already committed transactions. And in case of donor node reboot
+			 * xid<->csn mapping for them will be lost. However we must filter such
+			 * transactions in walreceiver before this code. --sk
+			 */
+			Assert(csn);
 			strncpy(gid, pq_getmsgstring(in), sizeof gid);
 			MTM_LOG2("PGLOGICAL_COMMIT_PREPARED commit: csn=%lld, gid=%s, lsn=%llx", csn, gid, end_lsn);
 			MtmResetTransaction();

t/004_recovery.pl

@@ -2,16 +2,54 @@
 use warnings;
 use Cluster;
 use TestLib;
-use Test::More tests => 2;
+use Test::More tests => 3;
 
 my $cluster = new Cluster(3);
 $cluster->init();
 $cluster->configure();
 $cluster->start();
 sleep(10);
 
-$cluster->psql(0, 'postgres', "create extension multimaster");
+$cluster->psql(0, 'postgres', "create extension multimaster;
+	create table if not exists t(k int primary key, v int);");
+
+$cluster->psql(0, 'postgres', "insert into t values(1, 10);");
+$cluster->psql(1, 'postgres', "insert into t values(2, 20);");
+$cluster->psql(2, 'postgres', "insert into t values(3, 30);");
+sleep(2);
+
+
+my $sum0;
+my $sum1;
+my $sum2;
+
+########################################################
+# Check start after all nodes were disconnected
+########################################################
+
+$cluster->{nodes}->[1]->stop('fast');
+$cluster->{nodes}->[2]->stop('fast');
+
+sleep(5);
+$cluster->{nodes}->[1]->start;
+# try to start node3 right here?
+sleep(5);
+$cluster->{nodes}->[2]->start;
+sleep(5);
+
+$cluster->psql(0, 'postgres', "select sum(v) from t;", stdout => \$sum0);
+$cluster->psql(1, 'postgres', "select sum(v) from t;", stdout => \$sum1);
+$cluster->psql(1, 'postgres', "select sum(v) from t;", stdout => \$sum2);
+is( (($sum0 == 60) and ($sum1 == $sum0) and ($sum2 == $sum0)) , 1, "Check that nodes are working and sync");
+
+########################################################
+# Check recovery during some load
+########################################################
+
 $cluster->pgbench(0, ('-i', -s => '10') );
+$cluster->pgbench(0, ('-N', -T => '1') );
+$cluster->pgbench(1, ('-N', -T => '1') );
+$cluster->pgbench(2, ('-N', -T => '1') );
 
 # kill node while neighbour is under load
 my $pgb_handle = $cluster->pgbench_async(1, ('-N', -T => '10') );
@@ -29,9 +67,6 @@
 sleep(10);
 
 # check data identity
-my $sum0;
-my $sum1;
-my $sum2;
 $cluster->psql(0, 'postgres', "select sum(abalance) from pgbench_accounts;", stdout => \$sum0);
 $cluster->psql(1, 'postgres', "select sum(abalance) from pgbench_accounts;", stdout => \$sum1);
 $cluster->psql(2, 'postgres', "select sum(abalance) from pgbench_accounts;", stdout => \$sum2);