and some even less obvious races in tests

Author: kelvich

Cluster.pm

@@ -73,6 +73,7 @@ sub new
 	my $self = {
 		nodenum => $nodenum,
 		nodes => $nodes,
+		recv_timeout => 5,
 	};
 
 	bless $self, $class;

t/001_basic_recovery.pl

@@ -36,6 +36,8 @@
 
 note("stopping node 2");
 $cluster->{nodes}->[2]->stop;
+
+sleep($cluster->{recv_timeout});
 $cluster->await_nodes( (0,1) );
 
 note("inserting 2 on node 0");
@@ -66,7 +68,8 @@
 note("starting node 2");
 $cluster->{nodes}->[2]->start;
 
-$cluster->await_nodes( (0,1,2) );
+# intentionaly start from 2
+$cluster->await_nodes( (2,0,1) );
 
 note("inserting 6 on node 0 (can fail)");
 $cluster->psql(0, 'postgres', "insert into t values(6, 60);"); 

t/004_recovery.pl

@@ -29,6 +29,7 @@
     (select * from pgbench_accounts order by aid) t;";
 
 $cluster->{nodes}->[2]->stop('fast');
+sleep($cluster->{recv_timeout});
 $cluster->await_nodes( (0,1) );
 
 $cluster->pgbench(0, ('-n','-N', -T => '4') );
@@ -42,7 +43,7 @@
 $oldhash = $hash0;
 
 $cluster->{nodes}->[2]->start;
-$cluster->await_nodes( (0,1,2) );
+$cluster->await_nodes( (2,0,1) );
 
 $cluster->psql(0, 'postgres', $hash_query, stdout => \$hash0);
 $cluster->psql(1, 'postgres', $hash_query, stdout => \$hash1);
@@ -70,7 +71,7 @@
 $cluster->{nodes}->[1]->start;
 $cluster->{nodes}->[2]->start;
 
-$cluster->await_nodes( (0,1,2) );
+$cluster->await_nodes( (1,2,0) );
 
 $cluster->psql(0, 'postgres', "select sum(v) from t;", stdout => \$sum0);
 $cluster->psql(1, 'postgres', "select sum(v) from t;", stdout => \$sum1);
@@ -99,7 +100,7 @@
 $cluster->pgbench_await($pgb_handle);
 
 # await recovery
-$cluster->await_nodes( (0,1,2) );
+$cluster->await_nodes( (2,0,1) );
 
 # check data identity
 $cluster->psql(0, 'postgres', $hash_query, stdout => \$hash0);

t/005_add_stop_node.pl

@@ -12,8 +12,7 @@
 
 # XXXX: delete all '-n' ?
 
-# await online status
-$cluster->{nodes}->[0]->poll_query_until('postgres', "select 't'");
+$cluster->await_nodes( (0,1,2) );
 
 # init
 $cluster->pgbench(0, ('-i', -s => '10') );
@@ -29,7 +28,8 @@
 $cluster->{nodes}->[2]->stop('fast');
 $cluster->pgbench(0, ('-N', '-n', -T => '1') );
 $cluster->{nodes}->[2]->start;
-$cluster->{nodes}->[2]->poll_query_until('postgres', "select 't'");
+
+$cluster->await_nodes( (2,0,1) );
 is($cluster->is_data_identic( (0,1,2) ), 1, "check auto recovery");
 
 # ################################################################################
@@ -51,7 +51,8 @@
 $cluster->psql(0, 'postgres', "SELECT mtm.add_node('$new_connstr')");
 # await for comletion?
 $cluster->{nodes}->[3]->start;
-$cluster->{nodes}->[3]->poll_query_until('postgres', "select 't'");
+
+$cluster->await_nodes( (3,0,1,2) );
 $cluster->pgbench(0, ('-N', '-n', -T => '1') );
 $cluster->pgbench(3, ('-N', '-n', -T => '1') );
 is($cluster->is_data_identic( (0,1,2,3) ), 1, "basebackup and add node");
@@ -88,6 +89,8 @@
 note('Stopping node with slot drop');
 # $cluster->psql(0, 'postgres', "select mtm.stop_node(3,'t')");
 $cluster->{nodes}->[2]->stop('fast');
+
+sleep($cluster->{recv_timeout});
 $cluster->await_nodes( (0,1,3) );
 
 $cluster->pgbench(0, ('-N', '-n', -T => '1') );
@@ -108,7 +111,7 @@
 note("preparing to start $dd");
 
 $cluster->{nodes}->[4]->start;
-$cluster->{nodes}->[4]->poll_query_until('postgres', "select 't'");
+$cluster->await_nodes( (4,0,1,3) );
 $cluster->pgbench(0, ('-N', '-n', -T => '1') );
 $cluster->pgbench(1, ('-N', '-n', -T => '1') );
 $cluster->pgbench(3, ('-N', '-n', -T => '1') );