add pgbench-based recovery test

Author: kelvich

Cluster.pm

@@ -10,6 +10,8 @@ use Cwd;
 
 use Socket;
 
+use IPC::Run;
+
 sub check_port
 {
 	my ($host, $port) = @_;
@@ -27,7 +29,6 @@ sub check_port
 	}
 
 	close(SOCK);
-	diag("checking for port $port = $available\n");
 	return $available;
 }
 
@@ -148,6 +149,7 @@ sub start
 	foreach my $node (@$nodes)
 	{
 		$node->start();
+		diag "Starting node with connstr 'dbname=postgres port=@{[ $node->port() ]} host=@{[ $node->host() ]}'";
 	}
 }
 
@@ -272,4 +274,37 @@ sub poll
 	return 0;
 }
 
+sub pgbench()
+{
+	my ($self, $node, @args) = @_;
+	my $pgbench_handle = $self->pgbench_async($node, @args);
+	$self->pgbench_await($pgbench_handle);
+}
+
+sub pgbench_async()
+{
+	my ($self, $node, @args) = @_;
+
+	my ($in, $out, $err, $rc);
+	$in = '';
+	$out = '';
+
+	my @pgbench_command = (
+		'pgbench',
+		@args,
+		-h => $self->{nodes}->[$node]->host(),
+		-p => $self->{nodes}->[$node]->port(),
+		'postgres',
+	);
+	# diag("running pgbench init");
+	my $handle = IPC::Run::start(\@pgbench_command, $in, $out);
+	return $handle;
+}
+
+sub pgbench_await()
+{
+	my ($self, $pgbench_handle) = @_;
+	IPC::Run::finish($pgbench_handle) || BAIL_OUT("pgbench exited with $?");
+}
+
 1;

t/004_recovery.pl

@@ -0,0 +1,41 @@
+use strict;
+use warnings;
+use Cluster;
+use TestLib;
+use Test::More tests => 2;
+
+my $cluster = new Cluster(3);
+$cluster->init();
+$cluster->configure();
+$cluster->start();
+sleep(10);
+
+$cluster->psql(0, 'postgres', "create extension multimaster");
+$cluster->pgbench(0, ('-i', -s => '10') );
+
+# kill node while neighbour is under load
+my $pgb_handle = $cluster->pgbench_async(0, ('-N', -T => '10') );
+sleep(5);
+$cluster->{nodes}->[2]->stop('fast');
+$cluster->pgbench_await($pgb_handle);
+
+# start node while neighbour is under load
+$pgb_handle = $cluster->pgbench_async(0, ('-N', -T => '10') );
+sleep(5);
+$cluster->{nodes}->[2]->start;
+$cluster->pgbench_await($pgb_handle);
+
+# give it 10s to recover
+sleep(10);
+
+# check data identity
+my $sum0;
+my $sum1;
+my $sum2;
+$cluster->psql(0, 'postgres', "select sum(abalance) from pgbench_accounts;", stdout => \$sum0);
+$cluster->psql(1, 'postgres', "select sum(abalance) from pgbench_accounts;", stdout => \$sum1);
+$cluster->psql(2, 'postgres', "select sum(abalance) from pgbench_accounts;", stdout => \$sum2);
+
+diag("Sums: $sum0, $sum1, $sum2");
+is($sum2, $sum0, "Check that sum_2 == sum_0");
+is($sum2, $sum1, "Check that sum_2 == sum_1");