brand new reconnection logic

Author: kelvich

multimaster.c

@@ -3850,7 +3850,7 @@ static bool MtmTwoPhaseCommit(MtmCurrentTrans* x)
 				StartTransactionCommand();
 				if (x->status == TRANSACTION_STATUS_ABORTED) { 
 					FinishPreparedTransaction(x->gid, false);
-					elog(ERROR, "Transaction %s is aborted by DTM", x->gid);
+					elog(ERROR, "Transaction aborted by DTM");
 				} else {
 					FinishPreparedTransaction(x->gid, true);
 				}

tests2/lib/bank_client.py

@@ -4,11 +4,13 @@
 import aiopg
 import random
 import psycopg2
+from psycopg2.extensions import *
 import time
 import datetime
 import copy
 import aioprocessing
 import multiprocessing
+import logging
 
 class MtmTxAggregate(object):
 
@@ -57,6 +59,7 @@ def keep_trying(tries, delay, method, name, *args, **kwargs):
 class MtmClient(object):
 
     def __init__(self, dsns, n_accounts=100000):
+        # logging.basicConfig(level=logging.DEBUG)
         self.n_accounts = n_accounts
         self.dsns = dsns
         self.aggregates = {}
@@ -129,21 +132,41 @@ def exec_tx(self, tx_block, aggname_prefix, conn_i):
         if conn_i not in self.aggregates:
             self.aggregates[conn_i] = {}
         agg = self.aggregates[conn_i][aggname_prefix] = MtmTxAggregate(aggname)
-        pool = yield from aiopg.create_pool(self.dsns[conn_i])
-        conn = yield from pool.acquire()
-        cur = yield from conn.cursor()
+        dsn = self.dsns[conn_i]
+
+        conn = cur = False
+
         while self.running:
             agg.start_tx()
+
             try:
-                # yield from cur.execute('commit')
+                if (not conn) or conn.closed:
+                        # enable_hstore tries to perform select from database
+                        # which in case of select's failure will lead to exception
+                        # and stale connection to the database
+                        conn = yield from aiopg.connect(dsn, enable_hstore=False)
+                        print("reconnected")
+
+                if (not cur) or cur.closed:
+                        cur = yield from conn.cursor()
+
+                # ROLLBACK tx after previous exception.
+                # Doing this here instead of except handler to stay inside try
+                # block.
+                status = yield from conn.get_transaction_status()
+                if status != TRANSACTION_STATUS_IDLE:
+                    yield from cur.execute('rollback')
+
                 yield from tx_block(conn, cur, agg)
                 agg.finish_tx('commit')
-            except psycopg2.OperationalError as e:
-                if not cur.closed:
-                    yield from cur.execute('rollback')
-                agg.finish_tx('operational_rollback')
+
             except psycopg2.Error as e:
-                agg.finish_tx(e.pgerror)
+                agg.finish_tx(str(e).strip())
+                # Give evloop some free time.
+                # In case of continuous excetions we can loop here without returning
+                # back to event loop and block it
+                yield from asyncio.sleep(0.01)
+
         print("We've count to infinity!")
 
     @asyncio.coroutine
@@ -199,14 +222,14 @@ def bgrun(self):
         self.evloop_process = multiprocessing.Process(target=self.run, args=())
         self.evloop_process.start()
 
-    def get_aggregates(self, print=True):
+    def get_aggregates(self, _print=True):
         self.parent_pipe.send('status')
         resp = self.parent_pipe.recv()
-        if print:
+        if _print:
             MtmClient.print_aggregates(resp)
         return resp
 
-    def clean_aggregates(self, print=True):
+    def clean_aggregates(self):
         self.parent_pipe.send('status')
         self.parent_pipe.recv()
 

tests2/test_recovery.py

@@ -7,6 +7,7 @@
 import subprocess
 import datetime
 import docker
+import warnings
 
 from lib.bank_client import MtmClient
 from lib.failure_injector import *
@@ -63,6 +64,7 @@ def setUpClass(self):
 
         # XXX: add normal wait here
         time.sleep(20)
+        print('started')
         self.client = MtmClient([
             "dbname=regression user=postgres host=127.0.0.1 port=15432",
             "dbname=regression user=postgres host=127.0.0.1 port=15433",
@@ -77,8 +79,11 @@ def tearDownClass(self):
         # XXX: check nodes data identity here
         subprocess.check_call(['docker-compose','down'])
 
+    def setUp(self):
+        warnings.simplefilter("ignore", ResourceWarning)
+
     def test_normal_operations(self):
-        print('### normal_operations ###')
+        print('### test_normal_operations ###')
 
         aggs_failure, aggs = self.performFailure(NoFailure())
 
@@ -90,7 +95,7 @@ def test_normal_operations(self):
 
 
     def test_node_partition(self):
-        print('### nodePartitionTest ###')
+        print('### test_node_partition ###')
 
         aggs_failure, aggs = self.performFailure(SingleNodePartition('node3'))
 
@@ -103,7 +108,7 @@ def test_node_partition(self):
 
 
     def test_edge_partition(self):
-        print('### edgePartitionTest ###')
+        print('### test_edge_partition ###')
 
         aggs_failure, aggs = self.performFailure(EdgePartition('node2', 'node3'))
 
@@ -114,13 +119,31 @@ def test_edge_partition(self):
         self.assertCommits(aggs)
         self.assertIsolation(aggs)
 
-        subprocess.check_call(['blockade','join'])
-        print("Node3 joined back")
+    def test_node_restart(self):
+        print('### test_node_restart ###')
+
+        time.sleep(3)
+
+        aggs_failure, aggs = self.performFailure(RestartNode('node3'))
 
-        for i in range(50):
-            time.sleep(3)
-            self.clients.print_agg()
+        self.assertCommits(aggs_failure[:2])
+        self.assertNoCommits(aggs_failure[2:])
+        self.assertIsolation(aggs_failure)
+
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
 
+    def test_node_crash(self):
+        print('### test_node_crash ###')
+
+        aggs_failure, aggs = self.performFailure(CrashRecoverNode('node3'))
+
+        self.assertCommits(aggs_failure[:2])
+        self.assertNoCommits(aggs_failure[2:])
+        self.assertIsolation(aggs_failure)
+
+        self.assertCommits(aggs)
+        self.assertIsolation(aggs)
 
 if __name__ == '__main__':
     unittest.main()