Fix rare EdgePartition test failure.

It sometimes excludes 1 instead of 3.

Author: arssher

tests/lib/test_helper.py

@@ -257,9 +257,9 @@ def performRandomFailure(self, node, wait=0, nodes_wait_for_commit=[], node_wait
         failure = FailureClass(node)
 
         log.info('simulating failure {} on node "{}"'.format(FailureClass.__name__, node))
-        return self.performFailure(failure, wait, nodes_wait_for_commit, node_wait_for_online, stop_load, nodes_assert_commit_during_failure)
+        return self.performFailure(failure, wait, nodes_wait_for_commit, nodes_wait_for_online, stop_load, nodes_assert_commit_during_failure)
 
-    def performFailure(self, failure, wait=0, nodes_wait_for_commit=[], node_wait_for_online=None, stop_load=False, nodes_assert_commit_during_failure=[]):
+    def performFailure(self, failure, wait=0, nodes_wait_for_commit=[], nodes_wait_for_online=[], stop_load=False, nodes_assert_commit_during_failure=[]):
 
         time.sleep(TEST_WARMING_TIME)
 
@@ -291,9 +291,9 @@ def performFailure(self, failure, wait=0, nodes_wait_for_commit=[], node_wait_fo
             self.client.get_aggregates(clean=False)
             self.client.stop()
 
-        if node_wait_for_online is not None:
+        for node_wait_for_online in nodes_wait_for_online:
             self.awaitOnline(node_wait_for_online)
-        else:
+        if (len(nodes_wait_for_online) == 0):
             time.sleep(TEST_RECOVERY_TIME)
 
         if stop_load:

tests/test_recovery_random.py

@@ -53,7 +53,7 @@ def test_random_disasters(self):
             aggs_failure, aggs = self.performRandomFailure(
                 f'node{node_number}',
                 nodes_wait_for_commit=[n for n in range(3)],
-                node_wait_for_online=f"dbname=regression user=postgres host={self.host_ip} port={port}",
+                nodes_wait_for_online=[f"dbname=regression user=postgres host={self.host_ip} port={port}"],
                 stop_load=True,
                 nodes_assert_commit_during_failure=
                 nodes_assert_commit_during_failure)
@@ -76,8 +76,17 @@ def test_edge_partition(self):
         log.info('### test_edge_partition ###')
 
         aggs_failure, aggs = self.performFailure(
-            EdgePartition('node1', 'node3'), node_wait_for_online=
-            f"dbname=regression user=postgres host={self.host_ip} port=15434",
+            EdgePartition('node1', 'node3'),
+            # clique selection picks up the min mask, so in 1-2-3 sausage 12
+            # will be eventually the live nodes. However, there is a small risk
+            # of 3 successfully voting for 23 before 1 understands what's going
+            # on, in which case 1 is put into recovery which doesn't finish in
+            # 10s of the test given that the load is not stopped. This actually
+            # happened in CI. To avoid test failure, wait for both 1 and 3 to be
+            # online.
+            nodes_wait_for_online=[
+                f"dbname=regression user=postgres host={self.host_ip} port=15434",
+                f"dbname=regression user=postgres host={self.host_ip} port=15432"],
             stop_load=True)
 
         self.assertTrue(('commit' in aggs_failure[0]['transfer']['finish']) or
@@ -94,8 +103,8 @@ def _test_single_failure(self):
 
         failure = CrashRecoverNode('node3')
         aggs_failure, aggs = self.performFailure(
-            failure, node_wait_for_online=
-            "dbname=regression user=postgres host=127.0.0.1 port=15434",
+            failure,
+            nodes_wait_for_online=["dbname=regression user=postgres host=127.0.0.1 port=15434"],
             stop_load=True)
 
         self.assertCommits(aggs_failure[:2])

tests/test_referee.py

@@ -76,7 +76,7 @@ def test_neighbor_restart(self):
         log.info('### test_neighbor_restart ###')
 
         aggs_failure, aggs = self.performFailure(
-            RestartNode('node2'), node_wait_for_online=self.dsns[1],
+            RestartNode('node2'), nodes_wait_for_online=[self.dsns[1]],
             stop_load=True)
 
         self.assertCommits(aggs_failure[:1])
@@ -90,8 +90,8 @@ def test_node_crash(self):
         log.info('### test_node_crash ###')
 
         aggs_failure, aggs = self.performFailure(
-            CrashRecoverNode('node2'), node_wait_for_online=
-            self.dsns[1],
+            CrashRecoverNode('node2'), nodes_wait_for_online=
+            [self.dsns[1]],
             stop_load=True)
 
         self.assertCommits(aggs_failure[:1])
@@ -105,8 +105,8 @@ def test_partition_referee(self):
         log.info('### test_partition_referee ###')
 
         aggs_failure, aggs = self.performFailure(
-            SingleNodePartition('node2'), node_wait_for_online=
-            self.dsns[1],
+            SingleNodePartition('node2'), nodes_wait_for_online=
+            [self.dsns[1]],
             stop_load=True)
 
         self.assertCommits(aggs_failure[:1])
@@ -123,8 +123,8 @@ def test_double_failure_referee(self):
         log.info('### test_double_failure_referee ###')
 
         aggs_failure, aggs = self.performFailure(
-            SingleNodePartition('node2'), node_wait_for_online=
-            self.dsns[1], stop_load=True)
+            SingleNodePartition('node2'), nodes_wait_for_online=
+            [self.dsns[1]], stop_load=True)
 
         self.assertCommits(aggs_failure[:1])
         self.assertNoCommits(aggs_failure[1:])
@@ -134,8 +134,8 @@ def test_double_failure_referee(self):
         self.assertIsolation(aggs)
 
         aggs_failure, aggs = self.performFailure(
-            SingleNodePartition('node1'), node_wait_for_online=
-            self.dsns[0], stop_load=True)
+            SingleNodePartition('node1'), nodes_wait_for_online=
+            [self.dsns[0]], stop_load=True)
 
         self.assertNoCommits(aggs_failure[:1])
         self.assertCommits(aggs_failure[1:])