wip. handle exceptions. #6

Author: kelvich

tests2/lib/bank_client.py

@@ -2,6 +2,7 @@
 import random
 from multiprocessing import Process, Value, Queue
 import time
+import sys
 from event_history import *
 
 class ClientCollection(object):
@@ -59,26 +60,36 @@ def history(self):
         return self._history
 
     def check_total(self):
-        conn = psycopg2.connect(self.connstr)
-        cur = conn.cursor();
+        conn, cur = self.connect()
+
         while self.run.value:
             event_id = self.history.register_start('total')
-            cur.execute('select sum(amount) from bank_test')
-            self.history.register_finish(event_id, 'commit')
 
-            res = cur.fetchone()
-            if res[0] != 0:
-                print("Isolation error, total = %d" % (res[0],))
-                raise BaseException
+            try:
+                cur.execute('select sum(amount) from bank_test')
+                res = cur.fetchone()
+                if res[0] != 0:
+                    print("Isolation error, total = %d" % (res[0],))
+                    raise BaseException
+            except psycopg2.InterfaceError:
+                print("Got error: ", sys.exc_info())
+                print("Reconnecting")
+                conn, cur = self.connect(reconnect=True)
+            except:
+                print("Got error: ", sys.exc_info())
+                self.history.register_finish(event_id, 'rollback')
+            else:
+                self.history.register_finish(event_id, 'commit')
+
 
         cur.close()
         conn.close()
 
     def transfer_money(self):
-        print(self.connstr)
-        conn = psycopg2.connect(self.connstr)
-        cur = conn.cursor()
-        
+        #conn = psycopg2.connect(self.connstr)
+        #cur = conn.cursor()
+        conn, cur = self.connect()
+
         i = 0
         while self.run.value:
             i += 1
@@ -88,39 +99,59 @@ def transfer_money(self):
 
             event_id = self.history.register_start('transfer')
 
-            cur.execute('''update bank_test
+            try:
+                cur.execute('''update bank_test
                     set amount = amount - %s
                     where uid = %s''',
                     (amount, from_uid))
-            cur.execute('''update bank_test
+                cur.execute('''update bank_test
                     set amount = amount + %s
                     where uid = %s''',
                     (amount, to_uid))
 
-            try:
                 conn.commit()
+            except psycopg2.DatabaseError:
+                print("Got error: ", sys.exc_info())
+                print("Reconnecting")
+                
+                self.history.register_finish(event_id, 'rollback')
+                conn, cur = self.connect(reconnect=True)
             except:
+                print("Got error: ", sys.exc_info())
                 self.history.register_finish(event_id, 'rollback')
             else:
                 self.history.register_finish(event_id, 'commit')
-            
+
         cur.close()
         conn.close()
 
-    def watchdog(self):
+    def connect(self, reconnect=False):
+        
         while self.run.value:
-            time.sleep(1)
-            print('watchdog: ', self.history.aggregate())
+            try:
+                conn = psycopg2.connect(self.connstr)
+                cur = conn.cursor()
+            except:
+                print("Got error: ", sys.exc_info())
+                if not reconnect:
+                    raise
+            else:
+                return conn, cur
+
+    # def watchdog(self):
+    #    while self.run.value:
+    #        time.sleep(1)
+    #        print('watchdog: ', self.history.aggregate())
 
     def start(self):
         self.transfer_process = Process(target=self.transfer_money, args=())
         self.transfer_process.start()
-        
+
         self.total_process = Process(target=self.check_total, args=())
         self.total_process.start()
 
-        self.total_process = Process(target=self.watchdog, args=())
-        self.total_process.start()
+        #self.total_process = Process(target=self.watchdog, args=())
+        #self.total_process.start()
 
         return
 
@@ -136,5 +167,3 @@ def cleanup(self):
         cur.close()
         conn.close()
 
-
-

tests2/test_recovery.py

@@ -19,26 +19,44 @@ def tearDown(self):
         self.clients[0].cleanup()
         subprocess.check_call(['blockade','join'])
 
-#    def test_0_normal_operation(self):
-#        print('normalOpsTest')
-#        time.sleep(5)
-#        
-#        for client in self.clients:
-#            agg = client.history.aggregate()
-#            print(agg)
-#            self.assertTrue(agg['tx']['commit'] > 0)
+    def test_0_normal_operation(self):
+        print('### normalOpsTest ###')
+        print('Waiting 5s to check operability')
+        time.sleep(5)
+        
+        for client in self.clients:
+            agg = client.history.aggregate()
+            print(agg)
+            self.assertTrue(agg['transfer']['commit'] > 0)
 
     def test_1_node_disconnect(self):
-        print('disconnectTest')
-        time.sleep(3)
+        print('### disconnectTest ###')
 
         subprocess.check_call(['blockade','partition','node3'])
-        print('---node3 out---')
-        time.sleep(15)
+        print('Node3 disconnected')
+
+        print('Waiting 12s to discover failure')
+        time.sleep(12)
+        for client in self.clients:
+            agg = client.history.aggregate()
+            print(agg)
+
+        print('Waiting 3s to check operability')
+        time.sleep(3)
+        for client in self.clients:
+            agg = client.history.aggregate()
+            print(agg)
 
         subprocess.check_call(['blockade','join'])
-        print('---node2 and node3 are back---')
-        time.sleep(5)
+        print('Node3 connected back')
+
+        print('Waiting 12s for catch-up')
+        time.sleep(12)
+
+        for client in self.clients:
+            agg = client.history.aggregate()
+            print(agg)
+
 
 if __name__ == '__main__':
     unittest.main()