Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit 2df4524

Browse files
kelvichkelvich
committed
add tests for manual node stop/recovery
1 parent f639b4c commit 2df4524

File tree

2 files changed

+202
-9
lines changed

2 files changed

+202
-9
lines changed

‎Cluster.pm

Lines changed: 96 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ sub new
6666
my $node = new PostgresNode("node$i", $host, $pgport);
6767
$node->{id} = $i;
6868
$node->{arbiter_port} = $arbiter_port;
69+
$node->{mmconnstr} = "${ \$node->connstr('postgres') } arbiter_port=${ \$node->{arbiter_port} }";
6970
push(@$nodes, $node);
7071
}
7172

@@ -89,47 +90,54 @@ sub init
8990
}
9091
}
9192

93+
sub all_connstrs
94+
{
95+
my ($self) = @_;
96+
my $nodes = $self->{nodes};
97+
return join(', ', map { "${ \$_->connstr('postgres') } arbiter_port=${ \$_->{arbiter_port} }" } @$nodes);
98+
}
99+
100+
92101
sub configure
93102
{
94103
my ($self) = @_;
95104
my $nodes = $self->{nodes};
96-
my $nnodes = scalar @{ $nodes };
97105

98-
my $connstr = join(', ', map { "${ \$_->connstr('postgres') } arbiter_port=${ \$_->{arbiter_port} }" } @$nodes);
106+
my $connstr = $self->all_connstrs();
99107

100108
foreach my $node (@$nodes)
101109
{
102110
my $id = $node->{id};
103111
my $host = $node->host;
104112
my $pgport = $node->port;
105113
my $arbiter_port = $node->{arbiter_port};
114+
my $unix_sock_dir = $ENV{PGHOST};
106115

107116
$node->append_conf("postgresql.conf", qq(
108117
log_statement = none
109118
listen_addresses = '$host'
110-
unix_socket_directories = ''
119+
unix_socket_directories = '$unix_sock_dir'
111120
port = $pgport
112121
max_prepared_transactions = 10
113122
max_connections = 10
114123
max_worker_processes = 100
115124
wal_level = logical
116-
max_wal_senders = 5
125+
max_wal_senders = 6
117126
wal_sender_timeout = 0
118127
default_transaction_isolation = 'repeatable read'
119-
max_replication_slots = 5
128+
max_replication_slots = 6
120129
shared_preload_libraries = 'multimaster'
121130
shared_buffers = 16MB
122131
123132
multimaster.arbiter_port = $arbiter_port
124133
multimaster.workers = 1
125134
multimaster.node_id = $id
126135
multimaster.conn_strings = '$connstr'
127-
multimaster.heartbeat_recv_timeout = 2050
136+
multimaster.heartbeat_recv_timeout = 1050
128137
multimaster.heartbeat_send_timeout = 250
129-
multimaster.max_nodes = $nnodes
130-
multimaster.ignore_tables_without_pk = true
138+
multimaster.max_nodes = 6
139+
multimaster.ignore_tables_without_pk = false
131140
multimaster.queue_size = 4194304
132-
multimaster.min_2pc_timeout = 150000
133141
log_line_prefix = '%t: '
134142
));
135143

@@ -307,4 +315,83 @@ sub pgbench_await()
307315
IPC::Run::finish($pgbench_handle) || BAIL_OUT("pgbench exited with $?");
308316
}
309317

318+
sub is_data_identic()
319+
{
320+
my ($self, @nodenums) = @_;
321+
my $checksum = '';
322+
323+
my $sql = "select md5('(' || string_agg(aid::text || ', ' || abalance::text , '),(') || ')')
324+
from (select * from pgbench_accounts order by aid) t;";
325+
326+
foreach my $i (@nodenums)
327+
{
328+
my $current_hash = '';
329+
$self->{nodes}->[$i]->psql('postgres', $sql, stdout => \$current_hash);
330+
if ($current_hash eq '')
331+
{
332+
diag("got empty hash from node $i");
333+
return 0;
334+
}
335+
if ($checksum eq '')
336+
{
337+
$checksum = $current_hash;
338+
}
339+
elsif ($checksum ne $current_hash)
340+
{
341+
diag("got different hashes: $checksum ang $current_hash");
342+
return 0;
343+
}
344+
}
345+
346+
diag($checksum);
347+
return 1;
348+
}
349+
350+
sub add_node()
351+
{
352+
my ($self, %params) = @_;
353+
my $pgport = defined $params{port} ? $params{port} : (allocate_ports('127.0.0.1', 1))[0];
354+
my $arbiter_port = defined $params{arbiter_port} ? $params{arbiter_port} : (allocate_ports('127.0.0.1', 1))[0];
355+
356+
my $connstrs;
357+
my $node_id;
358+
359+
if (defined $params{node_id})
360+
{
361+
$node_id = $params{node_id};
362+
$connstrs = $self->all_connstrs();
363+
}
364+
else
365+
{
366+
my $new_conn = ", dbname=postgres host=127.0.0.1 port=$pgport arbiter_port=$arbiter_port";
367+
$connstrs = $self->all_connstrs() . $new_conn;
368+
$node_id = scalar(@{$self->{nodes}}) + 1;
369+
}
370+
371+
my $node = PostgresNode->get_new_node("node${node_id}x");
372+
373+
$self->{nodes}->[0]->backup("backup_for_$node_id");
374+
# do init from backup before setting host, since init_from_backup() checks
375+
# it default value
376+
$node->init_from_backup($self->{nodes}->[0], "backup_for_$node_id");
377+
378+
$node->{_host} = '127.0.0.1';
379+
$node->{_port} = $pgport;
380+
$node->{arbiter_port} = $arbiter_port;
381+
$node->{mmconnstr} = "${ \$node->connstr('postgres') } arbiter_port=${ \$node->{arbiter_port} }";
382+
$node->append_conf("postgresql.conf", qq(
383+
multimaster.arbiter_port = $arbiter_port
384+
multimaster.conn_strings = '$connstrs'
385+
multimaster.node_id = $node_id
386+
port = $pgport
387+
));
388+
$node->append_conf("pg_hba.conf", qq(
389+
local replication all trust
390+
host replication all 127.0.0.1/32 trust
391+
host replication all ::1/128 trust
392+
));
393+
394+
push($self->{nodes}, $node);
395+
}
396+
310397
1;

‎t/0005_add_stop_node.pl

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
use strict;
2+
use warnings;
3+
use PostgresNode;
4+
use Cluster;
5+
use TestLib;
6+
use Test::More tests => 5;
7+
8+
my $cluster = new Cluster(3);
9+
$cluster->init();
10+
$cluster->configure();
11+
$cluster->start();
12+
13+
# XXXX: delete all '-n' ?
14+
15+
# await online status
16+
$cluster->{nodes}->[0]->poll_query_until('postgres', "select 't'");
17+
18+
# init
19+
$cluster->pgbench(0, ('-i', -s => '10') );
20+
$cluster->pgbench(0, ('-N', '-n', -T => '1') );
21+
$cluster->pgbench(1, ('-N', '-n', -T => '1') );
22+
$cluster->pgbench(2, ('-N', '-n', -T => '1') );
23+
$cluster->psql(0, 'postgres', "create extension multimaster");
24+
25+
26+
# ################################################################################
27+
# # auto recovery
28+
# ################################################################################
29+
$cluster->{nodes}->[2]->stop('fast');
30+
$cluster->pgbench(0, ('-N', '-n', -T => '1') );
31+
$cluster->{nodes}->[2]->start;
32+
$cluster->{nodes}->[2]->poll_query_until('postgres', "select 't'");
33+
is($cluster->is_data_identic( (0,1,2) ), 1, "check auto recovery");
34+
35+
# ################################################################################
36+
# # auto recovery with dead slots
37+
# ################################################################################
38+
39+
# ## TBD
40+
41+
# ################################################################################
42+
# # basebackup and add node
43+
# ################################################################################
44+
45+
my $new_connstr;
46+
47+
$cluster->add_node();
48+
$new_connstr = $cluster->{nodes}->[3]->{mmconnstr};
49+
$cluster->psql(0, 'postgres', "SELECT mtm.add_node('$new_connstr')");
50+
# await for comletion?
51+
$cluster->{nodes}->[3]->start;
52+
$cluster->{nodes}->[3]->poll_query_until('postgres', "select 't'");
53+
$cluster->pgbench(0, ('-N', '-n', -T => '1') );
54+
$cluster->pgbench(3, ('-N', '-n', -T => '1') );
55+
is($cluster->is_data_identic( (0,1,2,3) ), 1, "basebackup and add node");
56+
57+
# ################################################################################
58+
# # soft stop / resume
59+
# ################################################################################
60+
61+
my ($stopped_out, $stopped_err);
62+
63+
$cluster->psql(0, 'postgres', "select mtm.stop_node(3,'f')");
64+
# await for comletion?
65+
$cluster->pgbench(0, ('-N', '-n', -T => '1') );
66+
$cluster->pgbench(1, ('-N', '-n', -T => '1') );
67+
$cluster->pgbench(3, ('-N', '-n', -T => '1') );
68+
$cluster->{nodes}->[2]->psql('postgres', "select 't'",
69+
stdout => \$stopped_out, stderr => \$stopped_err);
70+
is($cluster->is_data_identic( (0,1,3) ), 1, "soft stop / resume");
71+
is($stopped_out eq '' && $stopped_err ne '', 1, "soft stop / resume");
72+
73+
$cluster->psql(0, 'postgres', "select mtm.resume_node(3)");
74+
$cluster->{nodes}->[2]->poll_query_until('postgres', "select 't'");
75+
$cluster->pgbench(2, ('-N', '-n', -T => '1') );
76+
is($cluster->is_data_identic( (0,1,2,3) ), 1, "soft stop / resume");
77+
78+
# ################################################################################
79+
# # hard stop / basebackup / recover
80+
# ################################################################################
81+
82+
# diag('Stopping node with slot drop');
83+
# $cluster->psql(0, 'postgres', "select mtm.stop_node(3,'t')");
84+
# # await for comletion?
85+
# $cluster->{nodes}->[2]->stop();
86+
87+
# $cluster->pgbench(0, ('-N', '-n', -T => '1') );
88+
# $cluster->pgbench(1, ('-N', '-n', -T => '1') );
89+
# $cluster->pgbench(3, ('-N', '-n', -T => '1') );
90+
# is($cluster->is_data_identic( (0,1,3) ), 1, "hard stop / resume");
91+
92+
# $cluster->psql(0, 'postgres', "select mtm.recover_node(3)");
93+
94+
# # now we need to perform backup from live node
95+
# $cluster->add_node(port => $cluster->{nodes}->[2]->{port},
96+
# arbiter_port => $cluster->{nodes}->[2]->{arbiter_port},
97+
# node_id => 3);
98+
# diag("preparing to start");
99+
# $cluster->{nodes}->[4]->start;
100+
# $cluster->{nodes}->[4]->poll_query_until('postgres', "select 't'");
101+
102+
# $cluster->pgbench(0, ('-N', '-n', -T => '1') );
103+
# $cluster->pgbench(1, ('-N', '-n', -T => '1') );
104+
# $cluster->pgbench(3, ('-N', '-n', -T => '1') );
105+
# $cluster->pgbench(4, ('-N', '-n', -T => '1') );
106+
# is($cluster->is_data_identic( (0,1,3,4) ), 1, "hard stop / resume");

0 commit comments

Comments
 (0)