add docker-based tests with two nodes, stub for major/referee tests

Author: kelvich

tests2/lib/test_helper.py

@@ -0,0 +1,61 @@
+import unittest
+import time
+import datetime
+
+TEST_WARMING_TIME = 5
+TEST_DURATION = 10
+TEST_RECOVERY_TIME = 30
+TEST_SETUP_TIME = 20
+TEST_STOP_DELAY = 5
+
+class TestHelper(object):
+
+    def assertIsolation(self, aggs):
+        isolated = True
+        for conn_id, agg in enumerate(aggs):
+            isolated = isolated and agg['sumtotal']['isolation'] == 0
+        if not isolated:
+            raise AssertionError('Isolation failure')
+
+    def assertCommits(self, aggs):
+        commits = True
+        for conn_id, agg in enumerate(aggs):
+            commits = commits and 'commit' in agg['transfer']['finish']
+        if not commits:
+            print('No commits during aggregation interval')
+            time.sleep(100000)
+            raise AssertionError('No commits during aggregation interval')
+
+    def assertNoCommits(self, aggs):
+        commits = True
+        for conn_id, agg in enumerate(aggs):
+            commits = commits and 'commit' in agg['transfer']['finish']
+        if commits:
+            raise AssertionError('There are commits during aggregation interval')
+
+    def performFailure(self, failure):
+
+        time.sleep(TEST_WARMING_TIME)
+         
+        print('Simulate failure at ',datetime.datetime.utcnow())
+
+        failure.start()
+
+        self.client.clean_aggregates()
+        print('Started failure at ',datetime.datetime.utcnow())
+
+        time.sleep(TEST_DURATION)
+
+        print('Getting aggs at ',datetime.datetime.utcnow())
+        aggs_failure = self.client.get_aggregates()
+
+
+        failure.stop()
+
+        print('Eliminate failure at ',datetime.datetime.utcnow())
+
+        self.client.clean_aggregates()
+        time.sleep(TEST_RECOVERY_TIME)
+        aggs = self.client.get_aggregates()
+
+        return (aggs_failure, aggs)

tests2/support/two_nodes.yml

@@ -0,0 +1,35 @@
+version: '2'
+
+services:
+
+  node1:
+    container_name: node1
+    build: ../..
+    privileged: true
+    ulimits:
+      core: 14294967296
+    environment:
+      POSTGRES_USER: 'pg'
+      POSTGRES_DB: 'regression'
+      NODE_ID: 1
+      CONNSTRS: >-
+        dbname=regression user=pg host=node1,
+        dbname=regression user=pg host=node2
+    ports:
+      - "15432:5432"
+
+  node2:
+    container_name: node2
+    build: ../..
+    privileged: true
+    ulimits:
+      core: 14294967296
+    environment:
+      POSTGRES_USER: 'pg'
+      POSTGRES_DB: 'regression'
+      NODE_ID: 2
+      CONNSTRS: >-
+        dbname=regression user=pg host=node1,
+        dbname=regression user=pg host=node2
+    ports:
+      - "15433:5432"

tests2/test_major.py

@@ -0,0 +1,74 @@
+#
+# Based on Aphyr's test for CockroachDB.
+#
+
+import unittest
+import time
+import subprocess
+import datetime
+import docker
+import warnings
+
+from lib.bank_client import MtmClient
+from lib.failure_injector import *
+from lib.test_helper import *
+
+
+class MajorTest(unittest.TestCase, TestHelper):
+
+    @classmethod
+    def setUpClass(cls):
+        subprocess.check_call(['docker-compose',
+            '-f', 'support/two_nodes.yml',
+            'up',
+            '--force-recreate',
+            '--build',
+            '-d'])
+
+        # XXX: add normal wait here
+        time.sleep(TEST_SETUP_TIME)
+
+        cls.client = MtmClient([
+            "dbname=regression user=postgres host=127.0.0.1 port=15432",
+            "dbname=regression user=postgres host=127.0.0.1 port=15433"
+        ], n_accounts=1000)
+        cls.client.bgrun()
+
+    @classmethod
+    def tearDownClass(cls):
+        print('tearDown')
+        cls.client.stop()
+
+        time.sleep(TEST_STOP_DELAY)
+
+        if not cls.client.is_data_identic():
+            raise AssertionError('Different data on nodes')
+
+        if cls.client.no_prepared_tx() != 0:
+            raise AssertionError('There are some uncommitted tx')
+
+        # XXX: check nodes data identity here
+        # subprocess.check_call(['docker-compose','down'])
+
+    def setUp(self):
+        warnings.simplefilter("ignore", ResourceWarning)
+        time.sleep(20)
+        print('Start new test at ',datetime.datetime.utcnow())
+
+    def tearDown(self):
+        print('Finish test at ',datetime.datetime.utcnow())
+
+    def test_node_partition(self):
+        print('### test_node_partition ###')
+
+        aggs_failure, aggs = self.performFailure(SingleNodePartition('node2'))
+
+        self.assertNoCommits(aggs_failure)
+        self.assertIsolation(aggs_failure)
+
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
+
+if __name__ == '__main__':
+    unittest.main()
+