Location via proxy:   [ UP ]  
[Report a bug]   [Manage cookies]                
Skip to content

Commit f83bad4

Browse files
kelvichkelvich
committed
some more tests
1 parent 41f6c18 commit f83bad4

File tree

3 files changed

+94
-46
lines changed

3 files changed

+94
-46
lines changed

‎tests2/lib/bank_client.py

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -167,15 +167,15 @@ def total_tx(self, conn, cur, agg):
167167
total = yield from cur.fetchone()
168168
if total[0] != 0:
169169
agg.isolation += 1
170-
print(self.oops)
171-
print('Isolation error, total = ', total[0])
172-
yield from cur.execute('select * from mtm.get_nodes_state()')
173-
nodes_state = yield from cur.fetchall()
174-
for i, col in enumerate(self.nodes_state_fields):
175-
print("%17s" % col, end="\t")
176-
for j in range(3):
177-
print("%19s" % nodes_state[j][i], end="\t")
178-
print("\n")
170+
# print(self.oops)
171+
# print('Isolation error, total = ', total[0])
172+
# yield from cur.execute('select * from mtm.get_nodes_state()')
173+
# nodes_state = yield from cur.fetchall()
174+
# for i, col in enumerate(self.nodes_state_fields):
175+
# print("%17s" % col, end="\t")
176+
# for j in range(3):
177+
# print("%19s" % nodes_state[j][i], end="\t")
178+
# print("\n")
179179

180180

181181
def run(self):
@@ -196,9 +196,8 @@ def bgrun(self):
196196
self.evloop_process = multiprocessing.Process(target=self.run, args=())
197197
self.evloop_process.start()
198198

199-
# XXX: rename to start/get aggregation
200-
# XXX: introduce periodic report from client
201-
def get_status(self, print=True):
199+
# XXX: introduce periodic report from client?
200+
def get_aggregates(self, print=True):
202201
# print('test: sending status request')
203202
self.parent_pipe.send('status')
204203
# print('test: awaitng status response')
@@ -208,6 +207,10 @@ def get_status(self, print=True):
208207
MtmClient.print_aggregates(resp)
209208
return resp
210209

210+
def clean_aggregates(self, print=True):
211+
self.parent_pipe.send('status')
212+
self.parent_pipe.recv()
213+
211214
def stop(self):
212215
self.running = False
213216
self.evloop_process.terminate()

‎tests2/lib/failure_injector.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ def __init__(self):
88
def container_exec(self, node, command):
99
exec_id = self.docker_api.exec_create(node, command, user='root')
1010
output = self.docker_api.exec_start(exec_id)
11-
print(command, ' -> ', output)
11+
# print(command, ' -> ', output)
1212

1313

1414
class SingleNodePartition(FailureInjector):
@@ -39,8 +39,8 @@ def start(self):
3939
self.container_exec(self.nodeA, "iptables -A OUTPUT -s {} -j DROP".format(self.nodeB) )
4040

4141
def stop(self):
42-
self.container_exec(self.nodeA, "iptables -D INPUT -j DROP")
43-
self.container_exec(self.nodeA, "iptables -D OUTPUT -j DROP")
42+
self.container_exec(self.nodeA, "iptables -D INPUT -s {} -j DROP".format(self.nodeB))
43+
self.container_exec(self.nodeA, "iptables -D OUTPUT -s {} -j DROP".format(self.nodeB))
4444

4545

4646
class RestartNode(FailureInjector):

‎tests2/test_recovery.py

Lines changed: 76 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -35,71 +35,116 @@ def tearDownClass(self):
3535
print('tearDown')
3636
self.client.stop()
3737
# XXX: check nodes data identity here
38-
3938
# subprocess.check_call(['docker-compose','down'])
4039

4140
def test_normal_operations(self):
4241
print('### normal_operations ###')
4342

44-
# nofailure here
45-
# clean aggregates
46-
self.client.get_status()
47-
43+
self.client.clean_aggregates()
4844
time.sleep(TEST_DURATION)
49-
aggs = self.client.get_status()
50-
51-
for agg in aggs:
52-
self.assertTrue( aggs[agg]['finish']['commit'] > 0 )
53-
54-
# nofailure ends here
55-
self.client.get_status()
45+
aggs_failure = self.client.get_aggregates()
5646

47+
self.client.clean_aggregates()
5748
time.sleep(TEST_RECOVERY_TIME)
58-
aggs = self.client.get_status()
49+
aggs = self.client.get_aggregates()
50+
51+
for agg in aggs_failure:
52+
self.assertTrue( 'commit' in aggs_failure[agg]['finish'] )
5953

6054
for agg in aggs:
61-
self.assertTrue( aggs[agg]['finish']['commit'] > 0 )
55+
self.assertTrue( 'commit' in aggs[agg]['finish'] )
6256

6357

6458
def test_node_partition(self):
6559
print('### nodePartitionTest ###')
6660

6761
failure = SingleNodePartition('node3')
68-
69-
# split one node
7062
failure.start()
71-
# clean aggregates
72-
self.client.get_status(print=False)
7363

64+
self.client.clean_aggregates()
7465
time.sleep(TEST_DURATION)
75-
aggs = self.client.get_status()
66+
aggs_failure = self.client.get_aggregates()
67+
68+
failure.stop()
7669

77-
self.assertTrue( aggs['transfer_0']['finish']['commit'] > 0 )
78-
self.assertTrue( aggs['transfer_1']['finish']['commit'] > 0 )
79-
self.assertTrue( 'commit' not in aggs['transfer_2']['finish'] )
70+
self.client.clean_aggregates()
71+
time.sleep(TEST_RECOVERY_TIME)
72+
aggs = self.client.get_aggregates()
73+
74+
self.assertTrue( 'commit' in aggs_failure['transfer_0']['finish'] )
75+
self.assertTrue( 'commit' in aggs_failure['transfer_1']['finish'] )
76+
self.assertTrue( 'commit' not in aggs_failure['transfer_2']['finish'] )
77+
self.assertTrue( aggs_failure['sumtotal_0']['isolation'] == 0)
78+
self.assertTrue( aggs_failure['sumtotal_1']['isolation'] == 0)
79+
self.assertTrue( aggs_failure['sumtotal_2']['isolation'] == 0)
80+
81+
self.assertTrue( 'commit' in aggs['transfer_0']['finish'] )
82+
self.assertTrue( 'commit' in aggs['transfer_1']['finish'] )
83+
self.assertTrue( 'commit' in aggs['transfer_2']['finish'] )
8084
self.assertTrue( aggs['sumtotal_0']['isolation'] == 0)
8185
self.assertTrue( aggs['sumtotal_1']['isolation'] == 0)
8286
self.assertTrue( aggs['sumtotal_2']['isolation'] == 0)
8387

84-
# join splitted node
88+
89+
def test_node_partition(self):
90+
print('### nodePartitionTest ###')
91+
92+
failure = SingleNodePartition('node3')
93+
failure.start()
94+
95+
self.client.clean_aggregates()
96+
time.sleep(TEST_DURATION)
97+
aggs_failure = self.client.get_aggregates()
98+
8599
failure.stop()
86-
# clear tx history
87-
self.client.get_status(print=False)
88100

101+
self.client.clean_aggregates()
89102
time.sleep(TEST_RECOVERY_TIME)
90-
aggs = self.client.get_status()
91-
92-
self.assertTrue( aggs['transfer_0']['finish']['commit'] > 0 )
93-
self.assertTrue( aggs['transfer_1']['finish']['commit'] > 0 )
94-
self.assertTrue( aggs['transfer_2']['finish']['commit'] > 0 )
103+
aggs = self.client.get_aggregates()
104+
105+
self.assertTrue( 'commit' in aggs_failure['transfer_0']['finish'] )
106+
self.assertTrue( 'commit' in aggs_failure['transfer_1']['finish'] )
107+
self.assertTrue( 'commit' not in aggs_failure['transfer_2']['finish'] )
108+
self.assertTrue( aggs_failure['sumtotal_0']['isolation'] == 0)
109+
self.assertTrue( aggs_failure['sumtotal_1']['isolation'] == 0)
110+
self.assertTrue( aggs_failure['sumtotal_2']['isolation'] == 0)
111+
112+
self.assertTrue( 'commit' in aggs['transfer_0']['finish'] )
113+
self.assertTrue( 'commit' in aggs['transfer_1']['finish'] )
114+
self.assertTrue( 'commit' in aggs['transfer_2']['finish'] )
95115
self.assertTrue( aggs['sumtotal_0']['isolation'] == 0)
96116
self.assertTrue( aggs['sumtotal_1']['isolation'] == 0)
97117
self.assertTrue( aggs['sumtotal_2']['isolation'] == 0)
98118

99-
# def test_edge_partition(self):
100119

120+
def test_edge_partition(self):
121+
print('### edgePartitionTest ###')
101122

123+
failure = EdgePartition('node2', 'node3')
124+
failure.start()
125+
126+
self.client.clean_aggregates()
127+
time.sleep(TEST_DURATION)
128+
aggs_failure = self.client.get_aggregates()
102129

130+
failure.stop()
131+
132+
self.client.clean_aggregates()
133+
time.sleep(TEST_RECOVERY_TIME)
134+
aggs = self.client.get_aggregates()
135+
136+
self.assertTrue( ('commit' in aggs_failure['transfer_0']['finish']) or ('commit' in aggs_failure['transfer_1']['finish']) )
137+
self.assertTrue( 'commit' not in aggs_failure['transfer_2']['finish'] )
138+
self.assertTrue( aggs_failure['sumtotal_0']['isolation'] == 0)
139+
self.assertTrue( aggs_failure['sumtotal_1']['isolation'] == 0)
140+
self.assertTrue( aggs_failure['sumtotal_2']['isolation'] == 0)
141+
142+
self.assertTrue( 'commit' in aggs['transfer_0']['finish'] )
143+
self.assertTrue( 'commit' in aggs['transfer_1']['finish'] )
144+
self.assertTrue( 'commit' in aggs['transfer_2']['finish'] )
145+
self.assertTrue( aggs['sumtotal_0']['isolation'] == 0)
146+
self.assertTrue( aggs['sumtotal_1']['isolation'] == 0)
147+
self.assertTrue( aggs['sumtotal_2']['isolation'] == 0)
103148

104149
subprocess.check_call(['blockade','join'])
105150
print("Node3 joined back")

0 commit comments

Comments
 (0)