@@ -20,7 +20,7 @@ def __init__(self, name):
2020def clear_values (self ):
2121self .max_latency = 0.0
2222self .finish = {}
23-
23+
2424def start_tx (self ):
2525self .start_time = datetime .datetime .now ()
2626
@@ -34,7 +34,7 @@ def finish_tx(self, name):
3434self .finish [name ]= 1
3535else :
3636self .finish [name ]+= 1
37-
37+
3838def as_dict (self ):
3939return {
4040'running_latency' : (datetime .datetime .now ()- self .start_time ).total_seconds (),
@@ -43,13 +43,24 @@ def as_dict(self):
4343'finish' :copy .deepcopy (self .finish )
4444 }
4545
46+ def keep_trying (tries ,delay ,method ,name ,* args ,** kwargs ):
47+ for t in range (tries ):
48+ try :
49+ return method (* args ,** kwargs )
50+ except Exception as e :
51+ if t == tries - 1 :
52+ raise Exception ("%s failed all %d tries" % (name ,tries ))from e
53+ print ("%s failed [%d of %d]: %s" % (name ,t + 1 ,tries ,str (e )))
54+ time .sleep (delay )
55+ raise Exception ("this should not happen" )
56+
4657class MtmClient (object ):
4758
4859def __init__ (self ,dsns ,n_accounts = 100000 ):
4960self .n_accounts = n_accounts
5061self .dsns = dsns
5162self .aggregates = {}
52- self .initdb ( )
63+ keep_trying ( 40 , 1 , self .initdb , 'self.initdb' )
5364self .running = True
5465self .nodes_state_fields = ["id" ,"disabled" ,"disconnected" ,"catchUp" ,"slotLag" ,
5566"avgTransDelay" ,"lastStatusChange" ,"oldestSnapshot" ,"SenderPid" ,
@@ -173,7 +184,7 @@ def run(self):
173184asyncio .async (self .status ())
174185
175186self .loop .run_forever ()
176-
187+
177188def bgrun (self ):
178189print ('Starting evloop in different process' );
179190self .parent_pipe ,self .child_pipe = aioprocessing .AioPipe ()
@@ -187,7 +198,7 @@ def get_status(self):
187198resp = self .parent_pipe .recv ()
188199print ('test: got status response' )
189200return resp
190-
201+
191202def stop (self ):
192203self .running = False
193204self .evloop_process .terminate ()