NotificationsYou must be signed in to change notification settings
Fork35
Star144

Commit28ac425

committed

Port numbers management is improved (#164)

- We don't release a port number that was defined by client- We only check log files to detect port number conflicts- We use slightly smarter log file checkingA test is added.

1 parent4fe1894 commit28ac425Copy full SHA for 28ac425

File tree

2 files changed

+189

-17

lines changed

testgres
- node.py
tests
- test_simple.py

2 files changed

+189

-17

lines changed

`‎testgres/node.py`

Lines changed: 72 additions & 17 deletions

Original file line number	Diff line number	Diff line change
`@@ -83,13 +83,13 @@`
`83`	`83`
`84`	`84`	`from .standbyimportFirst`
`85`	`85`
	`86`	`+from .importutils`
	`87`	`+`
`86`	`88`	`from .utilsimport \`
`87`	`89`	`PgVer, \`
`88`	`90`	`eprint, \`
`89`	`91`	`get_bin_path, \`
`90`	`92`	`get_pg_version, \`
`91`		`-reserve_port, \`
`92`		`-release_port, \`
`93`	`93`	`execute_utility, \`
`94`	`94`	`options_string, \`
`95`	`95`	`clean_on_error`
`@@ -158,7 +158,7 @@ def __init__(self, name=None, base_dir=None, port=None, conn_params: ConnectionP`
`158`	`158`	`self.os_ops=LocalOperations(conn_params)`
`159`	`159`
`160`	`160`	`self.host=self.os_ops.host`
`161`		`-self.port=portorreserve_port()`
	`161`	`+self.port=portorutils.reserve_port()`
`162`	`162`
`163`	`163`	`self.ssh_key=self.os_ops.ssh_key`
`164`	`164`
`@@ -471,6 +471,28 @@ def _collect_special_files(self):`
`471`	`471`
`472`	`472`	`returnresult`
`473`	`473`
	`474`	`+def_collect_log_files(self):`
	`475`	`+# dictionary of log files + size in bytes`
	`476`	`+`
	`477`	`+files= [`
	`478`	`+self.pg_log_file`
	`479`	`+ ]# yapf: disable`
	`480`	`+`
	`481`	`+result= {}`
	`482`	`+`
	`483`	`+forfinfiles:`
	`484`	`+# skip missing files`
	`485`	`+ifnotself.os_ops.path_exists(f):`
	`486`	`+continue`
	`487`	`+`
	`488`	`+file_size=self.os_ops.get_file_size(f)`
	`489`	`+asserttype(file_size)==int# noqa: E721`
	`490`	`+assertfile_size>=0`
	`491`	`+`
	`492`	`+result[f]=file_size`
	`493`	`+`
	`494`	`+returnresult`
	`495`	`+`
`474`	`496`	`definit(self,initdb_params=None,cached=True,**kwargs):`
`475`	`497`	`"""`
`476`	`498`	`Perform initdb for this node.`
`@@ -722,6 +744,22 @@ def slow_start(self, replica=False, dbname='template1', username=None, max_attem`
`722`	`744`	`OperationalError},`
`723`	`745`	`max_attempts=max_attempts)`
`724`	`746`
	`747`	`+def_detect_port_conflict(self,log_files0,log_files1):`
	`748`	`+asserttype(log_files0)==dict# noqa: E721`
	`749`	`+asserttype(log_files1)==dict# noqa: E721`
	`750`	`+`
	`751`	`+forfileinlog_files1.keys():`
	`752`	`+read_pos=0`
	`753`	`+`
	`754`	`+iffileinlog_files0.keys():`
	`755`	`+read_pos=log_files0[file]# the previous size`
	`756`	`+`
	`757`	`+file_content=self.os_ops.read_binary(file,read_pos)`
	`758`	`+file_content_s=file_content.decode()`
	`759`	`+if'Is another postmaster already running on port'infile_content_s:`
	`760`	`+returnTrue`
	`761`	`+returnFalse`
	`762`	`+`
`725`	`763`	`defstart(self,params=[],wait=True):`
`726`	`764`	`"""`
`727`	`765`	`Starts the PostgreSQL node using pg_ctl if node has not been started.`
`@@ -745,27 +783,42 @@ def start(self, params=[], wait=True):`
`745`	`783`	`"-w"ifwaitelse'-W',# --wait or --no-wait`
`746`	`784`	`"start"]+params# yapf: disable`
`747`	`785`
`748`		`-startup_retries=5`
	`786`	`+log_files0=self._collect_log_files()`
	`787`	`+asserttype(log_files0)==dict# noqa: E721`
	`788`	`+`
	`789`	`+nAttempt=0`
	`790`	`+timeout=1`
`749`	`791`	`whileTrue:`
	`792`	`+nAttempt+=1`
`750`	`793`	`try:`
`751`	`794`	`exit_status,out,error=execute_utility(_params,self.utils_log_file,verbose=True)`
`752`	`795`	`iferrorand'does not exist'inerror:`
`753`	`796`	`raiseException`
`754`	`797`	`exceptExceptionase:`
`755`		`-files=self._collect_special_files()`
`756`		`-ifany(len(file)>1and'Is another postmaster already '`
`757`		`-'running on port'infile[1].decode()for`
`758`		`-fileinfiles):`
`759`		`-logging.warning("Detected an issue with connecting to port {0}. "`
`760`		`-"Trying another port after a 5-second sleep...".format(self.port))`
`761`		`-self.port=reserve_port()`
`762`		`-options= {'port':str(self.port)}`
`763`		`-self.set_auto_conf(options)`
`764`		`-startup_retries-=1`
`765`		`-time.sleep(5)`
`766`		`-continue`
	`798`	`+ifself._should_free_portandnAttempt<5:`
	`799`	`+log_files1=self._collect_log_files()`
	`800`	`+ifself._detect_port_conflict(log_files0,log_files1):`
	`801`	`+log_files0=log_files1`
	`802`	`+logging.warning(`
	`803`	`+"Detected an issue with connecting to port {0}. "`
	`804`	`+"Trying another port after a {1}-second sleep...".format(self.port,timeout)`
	`805`	`+ )`
	`806`	`+time.sleep(timeout)`
	`807`	`+timeout=min(2*timeout,5)`
	`808`	`+cur_port=self.port`
	`809`	`+new_port=utils.reserve_port()# throw`
	`810`	`+try:`
	`811`	`+options= {'port':str(new_port)}`
	`812`	`+self.set_auto_conf(options)`
	`813`	`+except:# noqa: E722`
	`814`	`+utils.release_port(new_port)`
	`815`	`+raise`
	`816`	`+self.port=new_port`
	`817`	`+utils.release_port(cur_port)`
	`818`	`+continue`
`767`	`819`
`768`	`820`	`msg='Cannot start node'`
	`821`	`+files=self._collect_special_files()`
`769`	`822`	`raise_from(StartNodeException(msg,files),e)`
`770`	`823`	`break`
`771`	`824`	`self._maybe_start_logger()`
`@@ -930,8 +983,10 @@ def free_port(self):`
`930`	`983`	`"""`
`931`	`984`
`932`	`985`	`ifself._should_free_port:`
	`986`	`+port=self.port`
`933`	`987`	`self._should_free_port=False`
`934`		`-release_port(self.port)`
	`988`	`+self.port=None`
	`989`	`+utils.release_port(port)`
`935`	`990`
`936`	`991`	`defcleanup(self,max_attempts=3,full=False):`
`937`	`992`	`"""`

`‎tests/test_simple.py`

Lines changed: 117 additions & 0 deletions

Original file line number	Diff line number	Diff line change
`@@ -1064,6 +1064,123 @@ def test_the_same_port(self):`
`1064`	`1064`
`1065`	`1065`	`self.assertIn("Cannot start node",str(ctx.exception))`
`1066`	`1066`
	`1067`	`+classtagPortManagerProxy:`
	`1068`	`+sm_prev_testgres_reserve_port=None`
	`1069`	`+sm_prev_testgres_release_port=None`
	`1070`	`+`
	`1071`	`+sm_DummyPortNumber=None`
	`1072`	`+sm_DummyPortMaxUsage=None`
	`1073`	`+`
	`1074`	`+sm_DummyPortCurrentUsage=None`
	`1075`	`+sm_DummyPortTotalUsage=None`
	`1076`	`+`
	`1077`	`+def__init__(self,dummyPortNumber,dummyPortMaxUsage):`
	`1078`	`+asserttype(dummyPortNumber)==int# noqa: E721`
	`1079`	`+asserttype(dummyPortMaxUsage)==int# noqa: E721`
	`1080`	`+assertdummyPortNumber>=0`
	`1081`	`+assertdummyPortMaxUsage>=0`
	`1082`	`+`
	`1083`	`+assert__class__.sm_prev_testgres_reserve_portisNone`
	`1084`	`+assert__class__.sm_prev_testgres_release_portisNone`
	`1085`	`+asserttestgres.utils.reserve_port==testgres.utils.internal__reserve_port`
	`1086`	`+asserttestgres.utils.release_port==testgres.utils.internal__release_port`
	`1087`	`+`
	`1088`	`+__class__.sm_prev_testgres_reserve_port=testgres.utils.reserve_port`
	`1089`	`+__class__.sm_prev_testgres_release_port=testgres.utils.release_port`
	`1090`	`+`
	`1091`	`+testgres.utils.reserve_port=__class__._proxy__reserve_port`
	`1092`	`+testgres.utils.release_port=__class__._proxy__release_port`
	`1093`	`+`
	`1094`	`+asserttestgres.utils.reserve_port==__class__._proxy__reserve_port`
	`1095`	`+asserttestgres.utils.release_port==__class__._proxy__release_port`
	`1096`	`+`
	`1097`	`+__class__.sm_DummyPortNumber=dummyPortNumber`
	`1098`	`+__class__.sm_DummyPortMaxUsage=dummyPortMaxUsage`
	`1099`	`+`
	`1100`	`+__class__.sm_DummyPortCurrentUsage=0`
	`1101`	`+__class__.sm_DummyPortTotalUsage=0`
	`1102`	`+`
	`1103`	`+def__enter__(self):`
	`1104`	`+returnself`
	`1105`	`+`
	`1106`	`+def__exit__(self,type,value,traceback):`
	`1107`	`+assert__class__.sm_DummyPortCurrentUsage==0`
	`1108`	`+`
	`1109`	`+assert__class__.sm_prev_testgres_reserve_portisnotNone`
	`1110`	`+assert__class__.sm_prev_testgres_release_portisnotNone`
	`1111`	`+`
	`1112`	`+asserttestgres.utils.reserve_port==__class__._proxy__reserve_port`
	`1113`	`+asserttestgres.utils.release_port==__class__._proxy__release_port`
	`1114`	`+`
	`1115`	`+testgres.utils.reserve_port=__class__.sm_prev_testgres_reserve_port`
	`1116`	`+testgres.utils.release_port=__class__.sm_prev_testgres_release_port`
	`1117`	`+`
	`1118`	`+__class__.sm_prev_testgres_reserve_port=None`
	`1119`	`+__class__.sm_prev_testgres_release_port=None`
	`1120`	`+`
	`1121`	`+def_proxy__reserve_port():`
	`1122`	`+asserttype(__class__.sm_DummyPortMaxUsage)==int# noqa: E721`
	`1123`	`+asserttype(__class__.sm_DummyPortTotalUsage)==int# noqa: E721`
	`1124`	`+asserttype(__class__.sm_DummyPortCurrentUsage)==int# noqa: E721`
	`1125`	`+assert__class__.sm_DummyPortTotalUsage>=0`
	`1126`	`+assert__class__.sm_DummyPortCurrentUsage>=0`
	`1127`	`+`
	`1128`	`+assert__class__.sm_DummyPortTotalUsage<=__class__.sm_DummyPortMaxUsage`
	`1129`	`+assert__class__.sm_DummyPortCurrentUsage<=__class__.sm_DummyPortTotalUsage`
	`1130`	`+`
	`1131`	`+assert__class__.sm_prev_testgres_reserve_portisnotNone`
	`1132`	`+`
	`1133`	`+if__class__.sm_DummyPortTotalUsage==__class__.sm_DummyPortMaxUsage:`
	`1134`	`+return__class__.sm_prev_testgres_reserve_port()`
	`1135`	`+`
	`1136`	`+__class__.sm_DummyPortTotalUsage+=1`
	`1137`	`+__class__.sm_DummyPortCurrentUsage+=1`
	`1138`	`+return__class__.sm_DummyPortNumber`
	`1139`	`+`
	`1140`	`+def_proxy__release_port(dummyPortNumber):`
	`1141`	`+asserttype(dummyPortNumber)==int# noqa: E721`
	`1142`	`+`
	`1143`	`+asserttype(__class__.sm_DummyPortMaxUsage)==int# noqa: E721`
	`1144`	`+asserttype(__class__.sm_DummyPortTotalUsage)==int# noqa: E721`
	`1145`	`+asserttype(__class__.sm_DummyPortCurrentUsage)==int# noqa: E721`
	`1146`	`+assert__class__.sm_DummyPortTotalUsage>=0`
	`1147`	`+assert__class__.sm_DummyPortCurrentUsage>=0`
	`1148`	`+`
	`1149`	`+assert__class__.sm_DummyPortTotalUsage<=__class__.sm_DummyPortMaxUsage`
	`1150`	`+assert__class__.sm_DummyPortCurrentUsage<=__class__.sm_DummyPortTotalUsage`
	`1151`	`+`
	`1152`	`+assert__class__.sm_prev_testgres_release_portisnotNone`
	`1153`	`+`
	`1154`	`+if__class__.sm_DummyPortCurrentUsage>0anddummyPortNumber==__class__.sm_DummyPortNumber:`
	`1155`	`+assert__class__.sm_DummyPortTotalUsage>0`
	`1156`	`+__class__.sm_DummyPortCurrentUsage-=1`
	`1157`	`+return`
	`1158`	`+`
	`1159`	`+return__class__.sm_prev_testgres_release_port(dummyPortNumber)`
	`1160`	`+`
	`1161`	`+deftest_port_rereserve_during_node_start(self):`
	`1162`	`+C_COUNT_OF_BAD_PORT_USAGE=3`
	`1163`	`+`
	`1164`	`+withget_new_node()asnode1:`
	`1165`	`+node1.init().start()`
	`1166`	`+self.assertTrue(node1._should_free_port)`
	`1167`	`+self.assertEqual(type(node1.port),int)# noqa: E721`
	`1168`	`+node1.safe_psql("SELECT 1;")`
	`1169`	`+`
	`1170`	`+with__class__.tagPortManagerProxy(node1.port,C_COUNT_OF_BAD_PORT_USAGE):`
	`1171`	`+assert__class__.tagPortManagerProxy.sm_DummyPortNumber==node1.port`
	`1172`	`+withget_new_node()asnode2:`
	`1173`	`+self.assertTrue(node2._should_free_port)`
	`1174`	`+self.assertEqual(node2.port,node1.port)`
	`1175`	`+`
	`1176`	`+node2.init().start()`
	`1177`	`+`
	`1178`	`+self.assertNotEqual(node2.port,node1.port)`
	`1179`	`+self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortCurrentUsage,0)`
	`1180`	`+self.assertEqual(__class__.tagPortManagerProxy.sm_DummyPortTotalUsage,C_COUNT_OF_BAD_PORT_USAGE)`
	`1181`	`+`
	`1182`	`+node2.safe_psql("SELECT 1;")`
	`1183`	`+`
`1067`	`1184`	`deftest_simple_with_bin_dir(self):`
`1068`	`1185`	`withget_new_node()asnode:`
`1069`	`1186`	`node.init().start()`

0 commit comments

Comments

(0)

Movatterモバイル変換

Navigation Menu

Search code, repositories, users, issues, pull requests...

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Commit28ac425

File tree

2 files changed

2 files changed

`‎testgres/node.py`

`‎tests/test_simple.py`

0 commit comments