Aug 4, 2023 · Apr 10, 2023 · Apr 11, 2023 · Apr 11, 2023 · Apr 17, 2023 · May 2, 2023
diff --git a/testgres/node.py b/testgres/node.py
 import os
 import random
 import signal
 import subprocess
 import threading
 from queue import Queue

            exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True)
            if 'does not exist' in error:
                raise Exception
            if 'server started' in out:
                self.is_started = True
        except Exception as e:
            msg = 'Cannot start node'
            files = self._collect_special_files()
            raise_from(StartNodeException(msg, files), e)

        self._maybe_start_logger()
        self.is_started = True
        return self

    def stop(self, params=[], wait=True):

        # select query source
        if query:
            psql_params.extend(("-c", '"{}"'.format(query)))
            if self.os_ops.remote:
                psql_params.extend(("-c", '"{}"'.format(query)))
            else:
                psql_params.extend(("-c", query))
        elif filename:
            psql_params.extend(("-f", filename))
        else:
            raise QueryException('Query or filename must be provided')

        # should be the last one
        psql_params.append(dbname)
        if not self.os_ops.remote:
            # start psql process
            process = subprocess.Popen(psql_params,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)

            # wait until it finishes and get stdout and stderr
            out, err = process.communicate(input=input)
            return process.returncode, out, err
        else:
            status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

        # start psql process
        status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

        return status_code, out, err
            return status_code, out, err

    @method_decorator(positional_args_hack(['dbname', 'query']))
    def safe_psql(self, query=None, expect_error=False, **kwargs):
            err = e.message
        if ret:
            if expect_error:
                out = err or b''
                out =(err or b'').decode('utf-8')
            else:
                raise QueryException(err or b'', query)
                raise QueryException((err or b'').decode('utf-8'), query)
        elif expect_error:
            assert False, "Exception was expected, but query finished successfully: `{}` ".format(query)

                                         Defaults to an empty set.
        """
        # parse postgresql.auto.conf
        auto_conf_file = os.path.join(self.data_dir, config)
        raw_content = self.os_ops.read(auto_conf_file)
        path = os.path.join(self.data_dir, config)

        lines = self.os_ops.readlines(path)
        current_options = {}
        current_directives = []
        for line inraw_content.splitlines():
        for line inlines:

            # ignore comments
            if line.startswith('#'):
                continue

            if line == '':
            if line.strip() == '':
                continue

            if line.startswith('include'):
        for directive in current_directives:
            auto_conf += directive + "\n"

        self.os_ops.write(auto_conf_file, auto_conf)
        self.os_ops.write(path, auto_conf, truncate=True)


 class NodeApp:
diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py
 importgetpass
 importos
 importshutil
 importstat
 importsubprocess
 importtempfile
 fromshutilimportrmtree

 importpsutil

 from ..exceptionsimportExecUtilException

 from .os_opsimportOsOperations,ConnectionParams
 from .os_opsimportConnectionParams,OsOperations
 from .os_opsimportpglib

 try:
 fromdistutils.spawnimportfind_executable

 CMD_TIMEOUT_SEC=60
 error_markers= [b'error',b'Permission denied',b'fatal']


 classLocalOperations(OsOperations):
 def__init__(self,conn_params:ConnectionParams=ConnectionParams()):
 super().__init__(conn_params.username)
 def__init__(self,conn_params=None):
 ifconn_paramsisNone:
 conn_params=ConnectionParams()
 super(LocalOperations,self).__init__(conn_params.username)
 self.conn_params=conn_params
 self.host=conn_params.host
 self.ssh_key=None
 self.remote=False
 self.username=conn_params.usernameorself.get_user()

 # Command execution
 defexec_command(self,cmd,wait_exit=False,verbose=False,
 expect_error=False,encoding=None,shell=True,text=False,
 input=None,stdout=subprocess.PIPE,stderr=subprocess.PIPE,proc=None):
 expect_error=False,encoding=None,shell=False,text=False,
 input=None,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,proc=None):
 """
        Execute a command in a subprocess.

        - proc: The process to use for subprocess creation.
        :return: The output of the subprocess.
        """
 ifisinstance(cmd,list):
 cmd=' '.join(item.decode('utf-8')ifisinstance(item,bytes)elseitemforitemincmd)

 ifos.name=='nt':
 withtempfile.NamedTemporaryFile()asbuf:
 process=subprocess.Popen(cmd,stdout=buf,stderr=subprocess.STDOUT)
 result,error=process.communicate(input)
 exit_status=process.returncode

 found_error="error"inerror.decode(encodingor'utf-8').lower()
 error_found=exit_status!=0orany(markerinerrorformarkerinerror_markers)

 ifencoding:
 result=result.decode(encoding)
 ifexpect_error:
 raiseException(result,error)

 ifexit_status!=0orfound_error:
 ifexit_status!=0orerror_found:
 ifexit_status==0:
 exit_status=1
 raiseExecUtilException(message='Utility exited with non-zero code. Error `{}`'.format(error),

 defis_executable(self,file):
 # Check if the file is executable
 returnos.access(file,os.X_OK)
 returnos.stat(file).st_mode&stat.S_IXUSR

 defset_env(self,var_name,var_val):
 # Check if the directory is already in PATH

 # Work with dirs
 defmakedirs(self,path,remove_existing=False):
 ifremove_existingandos.path.exists(path):
 shutil.rmtree(path)
 os.makedirs(path,exist_ok=True)
 ifremove_existing:
 shutil.rmtree(path,ignore_errors=True)
 try:
 os.makedirs(path)
 exceptFileExistsError:
 pass

 defrmdirs(self,path,ignore_errors=True):
 returnrmtree(path,ignore_errors=ignore_errors)
 returnpathsep

 defmkdtemp(self,prefix=None):
 returntempfile.mkdtemp(prefix=prefix)
 returntempfile.mkdtemp(prefix='{}'.format(prefix))

 defmkstemp(self,prefix=None):
 fd,filename=tempfile.mkstemp(prefix=prefix)
diff --git a/testgres/operations/remote_ops.py b/testgres/operations/remote_ops.py
 sshtunnel.TUNNEL_TIMEOUT=5.0


 error_markers= [b'error',b'Permission denied']
 error_markers= [b'error',b'Permission denied',b'fatal']


 classPsUtilProcessProxy:
 self.host=conn_params.host
 self.ssh_key=conn_params.ssh_key
 self.ssh=self.ssh_connect()
 self.remote=True
 self.username=conn_params.usernameorself.get_user()
 self.tunnel=None

 ExecUtilException(message="An error occurred while reading the ssh key: {}".format(e))

 defexec_command(self,cmd:str,wait_exit=False,verbose=False,expect_error=False,
 encoding=None,shell=True,text=False,input=None,stdout=None,
 encoding=None,shell=True,text=False,input=None,stdin=None,stdout=None,
 stderr=None,proc=None):
 """
        Execute a command in the SSH session.
 iferror_found:
 ifexit_status==0:
 exit_status=1
 raiseExecUtilException(message="Utility exited with non-zero code. Error: {}".format(error.decode(encodingor'utf-8')),
 ifencoding:
 message="Utility exited with non-zero code. Error: {}".format(error.decode(encoding))
 else:
 message=b"Utility exited with non-zero code. Error: "+error
 raiseExecUtilException(message=message,
 command=cmd,
 exit_code=exit_status,
 out=result)
 conn=pglib.connect(
 host=host,# change to 'localhost' because we're connecting through a local ssh tunnel
 port=self.tunnel.local_bind_port,# use the local bind port set up by the tunnel
 dbname=dbname,
 database=dbname,
 user=userorself.username,
 password=password
            )
diff --git a/tests/test_remote.py b/tests/test_remote.py
 importos

 importpytest

 fromtestgresimportExecUtilException

 @pytest.fixture(scope="function",autouse=True)
 defsetup(self):
 conn_params=ConnectionParams(host="172.18.0.3",
 username="dev",
 ssh_key='../../container_files/postgres/ssh/id_ed25519')
 conn_params=ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST')or'172.18.0.3',
 username='dev',
 ssh_key=os.getenv(
 'RDBMS_TESTPOOL_SSHKEY')or'../../container_files/postgres/ssh/id_ed25519')
 self.operations=RemoteOperations(conn_params)

 yield
 exit_status,result,error=self.operations.exec_command(cmd,verbose=True,wait_exit=True)
 exceptExecUtilExceptionase:
 error=e.message
 asserterror=='Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n'
 asserterror==b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n'

 deftest_is_executable_true(self):
 """
 cmd="pwd"
 pwd=self.operations.exec_command(cmd,wait_exit=True,encoding='utf-8').strip()

 path=f"{pwd}/test_dir"
 path="{}/test_dir".format(pwd)

 # Test makedirs
 self.operations.makedirs(path)
 exit_status,result,error=self.operations.rmdirs(path,verbose=True)
 exceptExecUtilExceptionase:
 error=e.message
 asserterror=="Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n"
 asserterror==b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n"

 deftest_listdir(self):
 """
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,6 +3,7 @@
		import os
		import random
		import signal
		import subprocess
		import threading
		from queue import Queue

Expand DownExpand Up		@@ -714,14 +715,13 @@ def start(self, params=[], wait=True):
		exit_status, out, error = execute_utility(_params, self.utils_log_file, verbose=True)
		if 'does not exist' in error:
		raise Exception
		if 'server started' in out:
		self.is_started = True
		except Exception as e:
		msg = 'Cannot start node'
		files = self._collect_special_files()
		raise_from(StartNodeException(msg, files), e)

		self._maybe_start_logger()
		self.is_started = True
		return self

		def stop(self, params=[], wait=True):
Expand DownExpand Up		@@ -958,19 +958,31 @@ def psql(self,

		# select query source
		if query:
		psql_params.extend(("-c", '"{}"'.format(query)))
		if self.os_ops.remote:
		psql_params.extend(("-c", '"{}"'.format(query)))
		else:
		psql_params.extend(("-c", query))
		elif filename:
		psql_params.extend(("-f", filename))
		else:
		raise QueryException('Query or filename must be provided')

		# should be the last one
		psql_params.append(dbname)
		if not self.os_ops.remote:
		# start psql process
		process = subprocess.Popen(psql_params,
		stdin=subprocess.PIPE,
		stdout=subprocess.PIPE,
		stderr=subprocess.PIPE)

		# wait until it finishes and get stdout and stderr
		out, err = process.communicate(input=input)
		return process.returncode, out, err
		else:
		status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

		# start psql process
		status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

		return status_code, out, err
		return status_code, out, err

		@method_decorator(positional_args_hack(['dbname', 'query']))
		def safe_psql(self, query=None, expect_error=False, **kwargs):
Expand DownExpand Up		@@ -1002,9 +1014,9 @@ def safe_psql(self, query=None, expect_error=False, **kwargs):
		err = e.message
		if ret:
		if expect_error:
		out = err or b''
		out =(err or b'').decode('utf-8')
		else:
		raise QueryException(err or b'', query)
		raise QueryException((err or b'').decode('utf-8'), query)
		elif expect_error:
		assert False, "Exception was expected, but query finished successfully: `{}` ".format(query)

Expand DownExpand Up		@@ -1529,18 +1541,18 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}):
		Defaults to an empty set.
		"""
		# parse postgresql.auto.conf
		auto_conf_file = os.path.join(self.data_dir, config)
		raw_content = self.os_ops.read(auto_conf_file)
		path = os.path.join(self.data_dir, config)

		lines = self.os_ops.readlines(path)
		current_options = {}
		current_directives = []
		for line inraw_content.splitlines():
		for line inlines:

		# ignore comments
		if line.startswith('#'):
		continue

		if line == '':
		if line.strip() == '':
		continue

		if line.startswith('include'):
Expand DownExpand Up		@@ -1570,7 +1582,7 @@ def set_auto_conf(self, options, config='postgresql.auto.conf', rm_options={}):
		for directive in current_directives:
		auto_conf += directive + "\n"

		self.os_ops.write(auto_conf_file, auto_conf)
		self.os_ops.write(path, auto_conf, truncate=True)


		class NodeApp:
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,15 +1,15 @@
		importgetpass
		importos
		importshutil
		importstat
		importsubprocess
		importtempfile
		fromshutilimportrmtree

		importpsutil

		from ..exceptionsimportExecUtilException

		from .os_opsimportOsOperations,ConnectionParams
		from .os_opsimportConnectionParams,OsOperations
		from .os_opsimportpglib

		try:
Expand All		@@ -18,20 +18,24 @@
		fromdistutils.spawnimportfind_executable

		CMD_TIMEOUT_SEC=60
		error_markers= [b'error',b'Permission denied',b'fatal']


		classLocalOperations(OsOperations):
		def__init__(self,conn_params:ConnectionParams=ConnectionParams()):
		super().__init__(conn_params.username)
		def__init__(self,conn_params=None):
		ifconn_paramsisNone:
		conn_params=ConnectionParams()
		super(LocalOperations,self).__init__(conn_params.username)
		self.conn_params=conn_params
		self.host=conn_params.host
		self.ssh_key=None
		self.remote=False
		self.username=conn_params.usernameorself.get_user()

		# Command execution
		defexec_command(self,cmd,wait_exit=False,verbose=False,
		expect_error=False,encoding=None,shell=True,text=False,
		input=None,stdout=subprocess.PIPE,stderr=subprocess.PIPE,proc=None):
		expect_error=False,encoding=None,shell=False,text=False,
		input=None,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,proc=None):
		"""
		Execute a command in a subprocess.

Expand All		@@ -49,9 +53,6 @@ def exec_command(self, cmd, wait_exit=False, verbose=False,
		- proc: The process to use for subprocess creation.
		:return: The output of the subprocess.
		"""
		ifisinstance(cmd,list):
		cmd=' '.join(item.decode('utf-8')ifisinstance(item,bytes)elseitemforitemincmd)

		ifos.name=='nt':
		withtempfile.NamedTemporaryFile()asbuf:
		process=subprocess.Popen(cmd,stdout=buf,stderr=subprocess.STDOUT)
Expand All		@@ -71,7 +72,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False,
		result,error=process.communicate(input)
		exit_status=process.returncode

		found_error="error"inerror.decode(encodingor'utf-8').lower()
		error_found=exit_status!=0orany(markerinerrorformarkerinerror_markers)

		ifencoding:
		result=result.decode(encoding)
Expand All		@@ -80,7 +81,7 @@ def exec_command(self, cmd, wait_exit=False, verbose=False,
		ifexpect_error:
		raiseException(result,error)

		ifexit_status!=0orfound_error:
		ifexit_status!=0orerror_found:
		ifexit_status==0:
		exit_status=1
		raiseExecUtilException(message='Utility exited with non-zero code. Error `{}`'.format(error),
Expand All		@@ -101,7 +102,7 @@ def find_executable(self, executable):

		defis_executable(self,file):
		# Check if the file is executable
		returnos.access(file,os.X_OK)
		returnos.stat(file).st_mode&stat.S_IXUSR

		defset_env(self,var_name,var_val):
		# Check if the directory is already in PATH
Expand All		@@ -116,9 +117,12 @@ def get_name(self):

		# Work with dirs
		defmakedirs(self,path,remove_existing=False):
		ifremove_existingandos.path.exists(path):
		shutil.rmtree(path)
		os.makedirs(path,exist_ok=True)
		ifremove_existing:
		shutil.rmtree(path,ignore_errors=True)
		try:
		os.makedirs(path)
		exceptFileExistsError:
		pass

		defrmdirs(self,path,ignore_errors=True):
		returnrmtree(path,ignore_errors=ignore_errors)
Expand All		@@ -141,7 +145,7 @@ def pathsep(self):
		returnpathsep

		defmkdtemp(self,prefix=None):
		returntempfile.mkdtemp(prefix=prefix)
		returntempfile.mkdtemp(prefix='{}'.format(prefix))

		defmkstemp(self,prefix=None):
		fd,filename=tempfile.mkstemp(prefix=prefix)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -17,7 +17,7 @@
		sshtunnel.TUNNEL_TIMEOUT=5.0


		error_markers= [b'error',b'Permission denied']
		error_markers= [b'error',b'Permission denied',b'fatal']


		classPsUtilProcessProxy:
Expand All		@@ -43,6 +43,7 @@ def __init__(self, conn_params: ConnectionParams):
		self.host=conn_params.host
		self.ssh_key=conn_params.ssh_key
		self.ssh=self.ssh_connect()
		self.remote=True
		self.username=conn_params.usernameorself.get_user()
		self.tunnel=None

Expand DownExpand Up		@@ -89,7 +90,7 @@ def _read_ssh_key(self):
		ExecUtilException(message="An error occurred while reading the ssh key: {}".format(e))

		defexec_command(self,cmd:str,wait_exit=False,verbose=False,expect_error=False,
		encoding=None,shell=True,text=False,input=None,stdout=None,
		encoding=None,shell=True,text=False,input=None,stdin=None,stdout=None,
		stderr=None,proc=None):
		"""
		Execute a command in the SSH session.
Expand DownExpand Up		@@ -131,7 +132,11 @@ def exec_command(self, cmd: str, wait_exit=False, verbose=False, expect_error=Fa
		iferror_found:
		ifexit_status==0:
		exit_status=1
		raiseExecUtilException(message="Utility exited with non-zero code. Error: {}".format(error.decode(encodingor'utf-8')),
		ifencoding:
		message="Utility exited with non-zero code. Error: {}".format(error.decode(encoding))
		else:
		message=b"Utility exited with non-zero code. Error: "+error
		raiseExecUtilException(message=message,
		command=cmd,
		exit_code=exit_status,
		out=result)
Expand DownExpand Up		@@ -429,7 +434,7 @@ def db_connect(self, dbname, user, password=None, host="127.0.0.1", port=5432, s
		conn=pglib.connect(
		host=host,# change to 'localhost' because we're connecting through a local ssh tunnel
		port=self.tunnel.local_bind_port,# use the local bind port set up by the tunnel
		dbname=dbname,
		database=dbname,
		user=userorself.username,
		password=password
		)
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -1,3 +1,5 @@
		importos

		importpytest

		fromtestgresimportExecUtilException
Expand All		@@ -9,9 +11,10 @@ class TestRemoteOperations:

		@pytest.fixture(scope="function",autouse=True)
		defsetup(self):
		conn_params=ConnectionParams(host="172.18.0.3",
		username="dev",
		ssh_key='../../container_files/postgres/ssh/id_ed25519')
		conn_params=ConnectionParams(host=os.getenv('RDBMS_TESTPOOL1_HOST')or'172.18.0.3',
		username='dev',
		ssh_key=os.getenv(
		'RDBMS_TESTPOOL_SSHKEY')or'../../container_files/postgres/ssh/id_ed25519')
		self.operations=RemoteOperations(conn_params)

		yield
Expand All		@@ -35,7 +38,7 @@ def test_exec_command_failure(self):
		exit_status,result,error=self.operations.exec_command(cmd,verbose=True,wait_exit=True)
		exceptExecUtilExceptionase:
		error=e.message
		asserterror=='Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n'
		asserterror==b'Utility exited with non-zero code. Error: bash: line 1: nonexistent_command: command not found\n'

		deftest_is_executable_true(self):
		"""
Expand All		@@ -62,7 +65,7 @@ def test_makedirs_and_rmdirs_success(self):
		cmd="pwd"
		pwd=self.operations.exec_command(cmd,wait_exit=True,encoding='utf-8').strip()

		path=f"{pwd}/test_dir"
		path="{}/test_dir".format(pwd)

		# Test makedirs
		self.operations.makedirs(path)
Expand All		@@ -88,7 +91,7 @@ def test_makedirs_and_rmdirs_failure(self):
		exit_status,result,error=self.operations.rmdirs(path,verbose=True)
		exceptExecUtilExceptionase:
		error=e.message
		asserterror=="Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n"
		asserterror==b"Utility exited with non-zero code. Error: rm: cannot remove '/root/test_dir': Permission denied\n"

		deftest_listdir(self):
		"""
Expand Down