run recovery test at travis

Author: kelvich

tests2/lib/bank_client.py

@@ -30,7 +30,7 @@ def finish_tx(self, name):
         if latency > self.max_latency:
             self.max_latency = latency
 
-        if name not in self.finish: 
+        if name not in self.finish:
             self.finish[name] = 1
         else:
             self.finish[name] += 1
@@ -194,7 +194,7 @@ def run(self):
         self.loop.run_forever()
 
     def bgrun(self):
-        print('Starting evloop in different process');
+        print('Starting evloop in different process')
         self.parent_pipe, self.child_pipe = aioprocessing.AioPipe()
         self.evloop_process = multiprocessing.Process(target=self.run, args=())
         self.evloop_process.start()

tests2/lib/failure_injector.py

@@ -9,6 +9,13 @@ def container_exec(self, node, command):
         exec_id = self.docker_api.exec_create(node, command, user='root')
         output = self.docker_api.exec_start(exec_id)
 
+class NoFailure(FailureInjector):
+
+    def start(self):
+        return
+
+    def stop(self):
+        return
 
 class SingleNodePartition(FailureInjector):
 

tests2/test_recovery.py

@@ -57,12 +57,12 @@ class RecoveryTest(unittest.TestCase, TestHelper):
 
     @classmethod
     def setUpClass(self):
-        # subprocess.check_call(['docker-compose','up',
-        #     '--force-recreate',
-        #     '-d'])
+        subprocess.check_call(['docker-compose','up',
+            '--force-recreate',
+            '-d'])
 
         # XXX: add normal wait here
-        # time.sleep(30)
+        time.sleep(20)
         self.client = MtmClient([
             "dbname=regression user=postgres host=127.0.0.1 port=15432",
             "dbname=regression user=postgres host=127.0.0.1 port=15433",
@@ -75,24 +75,18 @@ def tearDownClass(self):
         print('tearDown')
         self.client.stop()
         # XXX: check nodes data identity here
-        # subprocess.check_call(['docker-compose','down'])
+        subprocess.check_call(['docker-compose','down'])
 
     def test_normal_operations(self):
         print('### normal_operations ###')
 
-        self.client.clean_aggregates()
-        time.sleep(TEST_DURATION)
-        aggs_failure = self.client.get_aggregates()
-
-        self.client.clean_aggregates()
-        time.sleep(TEST_RECOVERY_TIME)
-        aggs = self.client.get_aggregates()
+        aggs_failure, aggs = self.performFailure(NoFailure())
 
-        for agg in aggs_failure:
-            self.assertTrue( 'commit' in aggs_failure[agg]['finish'] )
+        self.assertCommits(aggs_failure)
+        self.assertIsolation(aggs_failure)
 
-        for agg in aggs:
-            self.assertTrue( 'commit' in aggs[agg]['finish'] )
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
 
 
     def test_node_partition(self):
@@ -111,31 +105,14 @@ def test_node_partition(self):
     def test_edge_partition(self):
         print('### edgePartitionTest ###')
 
-        failure = EdgePartition('node2', 'node3')
-        failure.start()
+        aggs_failure, aggs = self.performFailure(EdgePartition('node2', 'node3'))
 
-        self.client.clean_aggregates()
-        time.sleep(TEST_DURATION)
-        aggs_failure = self.client.get_aggregates()
-
-        failure.stop()
-
-        self.client.clean_aggregates()
-        time.sleep(TEST_RECOVERY_TIME)
-        aggs = self.client.get_aggregates()
+        self.assertTrue( ('commit' in aggs_failure[1]['transfer']['finish']) or ('commit' in aggs_failure[2]['transfer']['finish']) )
+        self.assertCommits(aggs_failure[0:1]) # first node
+        self.assertIsolation(aggs_failure)
 
-        self.assertTrue( ('commit' in aggs_failure['transfer_2']['finish']) or ('commit' in aggs_failure['transfer_1']['finish']) )
-        self.assertTrue( 'commit' in aggs_failure['transfer_0']['finish'] )
-        self.assertTrue( aggs_failure['sumtotal_0']['isolation'] == 0)
-        self.assertTrue( aggs_failure['sumtotal_1']['isolation'] == 0)
-        self.assertTrue( aggs_failure['sumtotal_2']['isolation'] == 0)
-
-        self.assertTrue( 'commit' in aggs['transfer_0']['finish'] )
-        self.assertTrue( 'commit' in aggs['transfer_1']['finish'] )
-        self.assertTrue( 'commit' in aggs['transfer_2']['finish'] )
-        self.assertTrue( aggs['sumtotal_0']['isolation'] == 0)
-        self.assertTrue( aggs['sumtotal_1']['isolation'] == 0)
-        self.assertTrue( aggs['sumtotal_2']['isolation'] == 0)
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
 
         subprocess.check_call(['blockade','join'])
         print("Node3 joined back")