Add failure detection and retries to the raftable test.

Author: kvap

contrib/raftable/t/000_basic.pl

@@ -68,16 +68,33 @@ sub start_nodes
 	world => genstr(3000),
 );
 
+my $retries = 100;
 my $timeout_ms = 1000;
 
-$able->psql('postgres', "select raftable('hello', '$tests{hello}', $timeout_ms);");
-$baker->psql('postgres', "select raftable('and', '$tests{and}', $timeout_ms);");
-$charlie->psql('postgres', "select raftable('goodbye', '$tests{goodbye}', $timeout_ms);");
+sub trysql {
+	my ($noderef, $sql) = @_;
+	while ($retries > 0) {
+		diag("try sql: ". substr($sql, 0, 60));
+		my ($rc, $stdout, $stderr) = $noderef->psql('postgres', $sql);
+		if (index($stderr, "after") == -1) {
+			return;
+		} else {
+			$retries--;
+			diag($stderr);
+			diag("psql failed, $retries retries left");
+		}
+	}
+	BAIL_OUT('no psql retries left');
+}
+
+trysql($able, "select raftable('hello', '$tests{hello}', $timeout_ms);");
+trysql($baker, "select raftable('and', '$tests{and}', $timeout_ms);");
+trysql($charlie, "select raftable('goodbye', '$tests{goodbye}', $timeout_ms);");
 $baker->stop;
-$able->psql('postgres', "select raftable('world', '$tests{world}', $timeout_ms);");
+trysql($able, "select raftable('world', '$tests{world}', $timeout_ms);");
 
 $baker->start;
-$baker->psql('postgres', "select raftable_sync($timeout_ms);");
+trysql($baker, "select raftable_sync($timeout_ms);");
 while (my ($key, $value) = each(%tests))
 {
 	my ($rc, $stdout, $stderr) = $baker->psql('postgres', "select raftable('$key');");