@@ -35,71 +35,116 @@ def tearDownClass(self):
3535print ('tearDown' )
3636self .client .stop ()
3737# XXX: check nodes data identity here
38-
3938# subprocess.check_call(['docker-compose','down'])
4039
4140def test_normal_operations (self ):
4241print ('### normal_operations ###' )
4342
44- # nofailure here
45- # clean aggregates
46- self .client .get_status ()
47-
43+ self .client .clean_aggregates ()
4844time .sleep (TEST_DURATION )
49- aggs = self .client .get_status ()
50-
51- for agg in aggs :
52- self .assertTrue (aggs [agg ]['finish' ]['commit' ]> 0 )
53-
54- # nofailure ends here
55- self .client .get_status ()
45+ aggs_failure = self .client .get_aggregates ()
5646
47+ self .client .clean_aggregates ()
5748time .sleep (TEST_RECOVERY_TIME )
58- aggs = self .client .get_status ()
49+ aggs = self .client .get_aggregates ()
50+
51+ for agg in aggs_failure :
52+ self .assertTrue ('commit' in aggs_failure [agg ]['finish' ] )
5953
6054for agg in aggs :
61- self .assertTrue (aggs [agg ]['finish' ][ 'commit' ] > 0 )
55+ self .assertTrue ('commit' in aggs [agg ]['finish' ] )
6256
6357
6458def test_node_partition (self ):
6559print ('### nodePartitionTest ###' )
6660
6761failure = SingleNodePartition ('node3' )
68-
69- # split one node
7062failure .start ()
71- # clean aggregates
72- self .client .get_status (print = False )
7363
64+ self .client .clean_aggregates ()
7465time .sleep (TEST_DURATION )
75- aggs = self .client .get_status ()
66+ aggs_failure = self .client .get_aggregates ()
67+
68+ failure .stop ()
7669
77- self .assertTrue (aggs ['transfer_0' ]['finish' ]['commit' ]> 0 )
78- self .assertTrue (aggs ['transfer_1' ]['finish' ]['commit' ]> 0 )
79- self .assertTrue ('commit' not in aggs ['transfer_2' ]['finish' ] )
70+ self .client .clean_aggregates ()
71+ time .sleep (TEST_RECOVERY_TIME )
72+ aggs = self .client .get_aggregates ()
73+
74+ self .assertTrue ('commit' in aggs_failure ['transfer_0' ]['finish' ] )
75+ self .assertTrue ('commit' in aggs_failure ['transfer_1' ]['finish' ] )
76+ self .assertTrue ('commit' not in aggs_failure ['transfer_2' ]['finish' ] )
77+ self .assertTrue (aggs_failure ['sumtotal_0' ]['isolation' ]== 0 )
78+ self .assertTrue (aggs_failure ['sumtotal_1' ]['isolation' ]== 0 )
79+ self .assertTrue (aggs_failure ['sumtotal_2' ]['isolation' ]== 0 )
80+
81+ self .assertTrue ('commit' in aggs ['transfer_0' ]['finish' ] )
82+ self .assertTrue ('commit' in aggs ['transfer_1' ]['finish' ] )
83+ self .assertTrue ('commit' in aggs ['transfer_2' ]['finish' ] )
8084self .assertTrue (aggs ['sumtotal_0' ]['isolation' ]== 0 )
8185self .assertTrue (aggs ['sumtotal_1' ]['isolation' ]== 0 )
8286self .assertTrue (aggs ['sumtotal_2' ]['isolation' ]== 0 )
8387
84- # join splitted node
88+
89+ def test_node_partition (self ):
90+ print ('### nodePartitionTest ###' )
91+
92+ failure = SingleNodePartition ('node3' )
93+ failure .start ()
94+
95+ self .client .clean_aggregates ()
96+ time .sleep (TEST_DURATION )
97+ aggs_failure = self .client .get_aggregates ()
98+
8599failure .stop ()
86- # clear tx history
87- self .client .get_status (print = False )
88100
101+ self .client .clean_aggregates ()
89102time .sleep (TEST_RECOVERY_TIME )
90- aggs = self .client .get_status ()
91-
92- self .assertTrue (aggs ['transfer_0' ]['finish' ]['commit' ]> 0 )
93- self .assertTrue (aggs ['transfer_1' ]['finish' ]['commit' ]> 0 )
94- self .assertTrue (aggs ['transfer_2' ]['finish' ]['commit' ]> 0 )
103+ aggs = self .client .get_aggregates ()
104+
105+ self .assertTrue ('commit' in aggs_failure ['transfer_0' ]['finish' ] )
106+ self .assertTrue ('commit' in aggs_failure ['transfer_1' ]['finish' ] )
107+ self .assertTrue ('commit' not in aggs_failure ['transfer_2' ]['finish' ] )
108+ self .assertTrue (aggs_failure ['sumtotal_0' ]['isolation' ]== 0 )
109+ self .assertTrue (aggs_failure ['sumtotal_1' ]['isolation' ]== 0 )
110+ self .assertTrue (aggs_failure ['sumtotal_2' ]['isolation' ]== 0 )
111+
112+ self .assertTrue ('commit' in aggs ['transfer_0' ]['finish' ] )
113+ self .assertTrue ('commit' in aggs ['transfer_1' ]['finish' ] )
114+ self .assertTrue ('commit' in aggs ['transfer_2' ]['finish' ] )
95115self .assertTrue (aggs ['sumtotal_0' ]['isolation' ]== 0 )
96116self .assertTrue (aggs ['sumtotal_1' ]['isolation' ]== 0 )
97117self .assertTrue (aggs ['sumtotal_2' ]['isolation' ]== 0 )
98118
99- # def test_edge_partition(self):
100119
120+ def test_edge_partition (self ):
121+ print ('### edgePartitionTest ###' )
101122
123+ failure = EdgePartition ('node2' ,'node3' )
124+ failure .start ()
125+
126+ self .client .clean_aggregates ()
127+ time .sleep (TEST_DURATION )
128+ aggs_failure = self .client .get_aggregates ()
102129
130+ failure .stop ()
131+
132+ self .client .clean_aggregates ()
133+ time .sleep (TEST_RECOVERY_TIME )
134+ aggs = self .client .get_aggregates ()
135+
136+ self .assertTrue ( ('commit' in aggs_failure ['transfer_0' ]['finish' ])or ('commit' in aggs_failure ['transfer_1' ]['finish' ]) )
137+ self .assertTrue ('commit' not in aggs_failure ['transfer_2' ]['finish' ] )
138+ self .assertTrue (aggs_failure ['sumtotal_0' ]['isolation' ]== 0 )
139+ self .assertTrue (aggs_failure ['sumtotal_1' ]['isolation' ]== 0 )
140+ self .assertTrue (aggs_failure ['sumtotal_2' ]['isolation' ]== 0 )
141+
142+ self .assertTrue ('commit' in aggs ['transfer_0' ]['finish' ] )
143+ self .assertTrue ('commit' in aggs ['transfer_1' ]['finish' ] )
144+ self .assertTrue ('commit' in aggs ['transfer_2' ]['finish' ] )
145+ self .assertTrue (aggs ['sumtotal_0' ]['isolation' ]== 0 )
146+ self .assertTrue (aggs ['sumtotal_1' ]['isolation' ]== 0 )
147+ self .assertTrue (aggs ['sumtotal_2' ]['isolation' ]== 0 )
103148
104149subprocess .check_call (['blockade' ,'join' ])
105150print ("Node3 joined back" )