Dec 10, 2024 · Dec 8, 2024 · Dec 9, 2024 · Dec 9, 2024 · Dec 9, 2024 · Dec 9, 2024
diff --git a/testgres/__init__.py b/testgres/__init__.py
 CatchUpException, \
 StartNodeException, \
 InitNodeException, \
 BackupException
 BackupException, \
 InvalidOperationException

 from .enumsimport \
 XLogMethod, \
 "NodeBackup","testgres_config",
 "TestgresConfig","configure_testgres","scoped_config","push_config","pop_config",
 "NodeConnection","DatabaseError","InternalError","ProgrammingError","OperationalError",
 "TestgresException","ExecUtilException","QueryException","TimeoutException","CatchUpException","StartNodeException","InitNodeException","BackupException",
 "TestgresException","ExecUtilException","QueryException","TimeoutException","CatchUpException","StartNodeException","InitNodeException","BackupException","InvalidOperationException",
 "XLogMethod","IsolationLevel","NodeStatus","ProcessType","DumpFormat",
 "PostgresNode","NodeApp",
 "reserve_port","release_port","bound_ports","get_bin_path","get_pg_config","get_pg_version",
diff --git a/testgres/exceptions.py b/testgres/exceptions.py

 @six.python_2_unicode_compatible
 classExecUtilException(TestgresException):
 def__init__(self,message=None,command=None,exit_code=0,out=None):
 def__init__(self,message=None,command=None,exit_code=0,out=None,error=None):
 super(ExecUtilException,self).__init__(message)

 self.message=message
 self.command=command
 self.exit_code=exit_code
 self.out=out
 self.error=error

 def__str__(self):
 msg= []
 msg.append(self.message)

 ifself.command:
 msg.append(u'Command: {}'.format(self.command))
 command_s=' '.join(self.command)ifisinstance(self.command,list)elseself.command,
 msg.append(u'Command: {}'.format(command_s))

 ifself.exit_code:
 msg.append(u'Exit code: {}'.format(self.exit_code))

 ifself.error:
 msg.append(u'---- Error:\n{}'.format(self.error))

 ifself.out:
 msg.append(u'----\n{}'.format(self.out))
 msg.append(u'---- Out:\n{}'.format(self.out))

 returnself.convert_and_join(msg)


 classBackupException(TestgresException):
 pass


 classInvalidOperationException(TestgresException):
 pass
diff --git a/testgres/node.py b/testgres/node.py
    TimeoutException,   \
    InitNodeException,  \
    TestgresException,  \
    BackupException
    BackupException,    \
    InvalidOperationException

 from .logger import TestgresLogger

            >>> psql(query='select 3', ON_ERROR_STOP=1)
        """

        return self._psql(
            ignore_errors=True,
            query=query,
            filename=filename,
            dbname=dbname,
            username=username,
            input=input,
            **variables
        )

    def _psql(
            self,
            ignore_errors,
            query=None,
            filename=None,
            dbname=None,
            username=None,
            input=None,
            **variables):
        assert type(variables) == dict  # noqa: E721

        #
        # We do not support encoding. It may be added later. Ok?
        #
        if input is None:
            pass
        elif type(input) == bytes:  # noqa: E721
            pass
        else:
            raise Exception("Input data must be None or bytes.")

        dbname = dbname or default_dbname()

        psql_params = [

        # should be the last one
        psql_params.append(dbname)
        if not self.os_ops.remote:
            # start psql process
            process = subprocess.Popen(psql_params,
                                       stdin=subprocess.PIPE,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)

            # wait until it finishes and get stdout and stderr
            out, err = process.communicate(input=input)
            return process.returncode, out, err
        else:
            status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

            return status_code, out, err
        return self.os_ops.exec_command(
            psql_params,
            verbose=True,
            input=input,
            stderr=subprocess.PIPE,
            stdout=subprocess.PIPE,
            ignore_errors=ignore_errors)

    @method_decorator(positional_args_hack(['dbname', 'query']))
    def safe_psql(self, query=None, expect_error=False, **kwargs):
        Returns:
            psql's output as str.
        """
        assert type(kwargs) == dict  # noqa: E721
        assert not ("ignore_errors" in kwargs.keys())
        assert not ("expect_error" in kwargs.keys())

        # force this setting
        kwargs['ON_ERROR_STOP'] = 1
        try:
            ret, out, err = self.psql(query=query, **kwargs)
            ret, out, err = self._psql(ignore_errors=False,query=query, **kwargs)
        except ExecUtilException as e:
            ret = e.exit_code
            out = e.out
            err = e.message
        if ret:
            if expect_error:
                out = (err or b'').decode('utf-8')
            else:
                raise QueryException((err or b'').decode('utf-8'), query)
        elif expect_error:
            assert False, "Exception was expected, but query finished successfully: `{}` ".format(query)
            if not expect_error:
                raise QueryException(e.message, query)

            if type(e.error) == bytes:  # noqa: E721
                return e.error.decode("utf-8")  # throw

            # [2024-12-09] This situation is not expected
            assert False
            return e.error

        if expect_error:
            raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query))

        return out

diff --git a/testgres/operations/helpers.py b/testgres/operations/helpers.py
 importlocale


 classHelpers:
 def_make_get_default_encoding_func():
 # locale.getencoding is added in Python 3.11
 ifhasattr(locale,'getencoding'):
 returnlocale.getencoding

 # It must exist
 returnlocale.getpreferredencoding

 # Prepared pointer on function to get a name of system codepage
 _get_default_encoding_func=_make_get_default_encoding_func()

 defGetDefaultEncoding():
 #
 #   Original idea/source was:
 #
 #   def os_ops.get_default_encoding():
 #       if not hasattr(locale, 'getencoding'):
 #       locale.getencoding = locale.getpreferredencoding
 #       return locale.getencoding() or 'UTF-8'
 #

 assert__class__._get_default_encoding_funcisnotNone

 r=__class__._get_default_encoding_func()

 ifr:
 assertrisnotNone
 asserttype(r)==str# noqa: E721
 assertr!=""
 returnr

 # Is it an unexpected situation?
 return'UTF-8'

 defPrepareProcessInput(input,encoding):
 ifnotinput:
 returnNone

 iftype(input)==str:# noqa: E721
 ifencodingisNone:
 returninput.encode(__class__.GetDefaultEncoding())

 asserttype(encoding)==str# noqa: E721
 returninput.encode(encoding)

 # It is expected!
 asserttype(input)==bytes# noqa: E721
 returninput
diff --git a/testgres/operations/local_ops.py b/testgres/operations/local_ops.py

 from ..exceptionsimportExecUtilException
 from .os_opsimportConnectionParams,OsOperations,pglib,get_default_encoding
 from .raise_errorimportRaiseError
 from .helpersimportHelpers

 try:
 fromshutilimportwhichasfind_executable
 self.remote=False
 self.username=conn_params.usernameorgetpass.getuser()

 @staticmethod
 def_raise_exec_exception(message,command,exit_code,output):
 """Raise an ExecUtilException."""
 raiseExecUtilException(message=message.format(output),
 command=' '.join(command)ifisinstance(command,list)elsecommand,
 exit_code=exit_code,
 out=output)

 @staticmethod
 def_process_output(encoding,temp_file_path):
 """Process the output of a command from a temporary file."""
 returnoutput,None# In Windows stderr writing in stdout

 def_run_command__nt(self,cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding):
 # TODO: why don't we use the data from input?

 withtempfile.NamedTemporaryFile(mode='w+b',delete=False)astemp_file:
 stdout=temp_file
 stderr=subprocess.STDOUT
 returnprocess,output,error

 def_run_command__generic(self,cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding):
 input_prepared=None
 ifnotget_process:
 input_prepared=Helpers.PrepareProcessInput(input,encoding)# throw

 assertinput_preparedisNoneor (type(input_prepared)==bytes)# noqa: E721

 process=subprocess.Popen(
 cmd,
 shell=shell,
 stdin=stdinorsubprocess.PIPEifinputisnotNoneelseNone,
 stdout=stdoutorsubprocess.PIPE,
 stderr=stderrorsubprocess.PIPE,
        )
 assertnot (processisNone)
 ifget_process:
 returnprocess,None,None
 try:
 output,error=process.communicate(input=input.encode(encoding)ifinputelseNone,timeout=timeout)
 ifencoding:
 output=output.decode(encoding)
 error=error.decode(encoding)
 returnprocess,output,error
 output,error=process.communicate(input=input_prepared,timeout=timeout)
 exceptsubprocess.TimeoutExpired:
 process.kill()
 raiseExecUtilException("Command timed out after {} seconds.".format(timeout))

 asserttype(output)==bytes# noqa: E721
 asserttype(error)==bytes# noqa: E721

 ifencoding:
 output=output.decode(encoding)
 error=error.decode(encoding)
 returnprocess,output,error

 def_run_command(self,cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding):
 """Execute a command and return the process and its output."""
 ifos.name=='nt'andstdoutisNone:# Windows
 """
        Execute a command in a subprocess and handle the output based on the provided parameters.
        """
 asserttype(expect_error)==bool# noqa: E721
 asserttype(ignore_errors)==bool# noqa: E721

 process,output,error=self._run_command(cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding)
 ifget_process:
 returnprocess
 ifnotignore_errorsand ((process.returncode!=0orhas_errors(output=output,error=error))andnotexpect_error):
 self._raise_exec_exception('Utility exited with non-zero code. Error `{}`',cmd,process.returncode,errororoutput)
 RaiseError.UtilityExitedWithNonZeroCode(
 cmd=cmd,
 exit_code=process.returncode,
 msg_arg=errororoutput,
 error=error,
 out=output
            )

 ifverbose:
 returnprocess.returncode,output,error
diff --git a/testgres/operations/raise_error.py b/testgres/operations/raise_error.py
 from ..exceptions import ExecUtilException
 from .helpers import Helpers


 class RaiseError:
    def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out):
        assert type(exit_code) == int  # noqa: E721

        msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip()
        assert type(msg_arg_s) == str  # noqa: E721

        if msg_arg_s == "":
            msg_arg_s = "#no_error_message"

        message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`"
        raise ExecUtilException(
            message=message,
            command=cmd,
            exit_code=exit_code,
            out=out,
            error=error)

    def _TranslateDataIntoString(data):
        if type(data) == bytes:  # noqa: E721
            return __class__._TranslateDataIntoString__FromBinary(data)

        return str(data)

    def _TranslateDataIntoString__FromBinary(data):
        assert type(data) == bytes  # noqa: E721

        try:
            return data.decode(Helpers.GetDefaultEncoding())
        except UnicodeDecodeError:
            pass

        return "#cannot_decode_text"

    def _BinaryIsASCII(data):
        assert type(data) == bytes  # noqa: E721

        for b in data:
            if not (b >= 0 and b <= 127):
                return False

        return True
Original file line number	Diff line number	Diff line change
Expand Up		@@ -23,7 +23,8 @@
		CatchUpException, \
		StartNodeException, \
		InitNodeException, \
		BackupException
		BackupException, \
		InvalidOperationException

		from .enumsimport \
		XLogMethod, \
Expand DownExpand Up		@@ -60,7 +61,7 @@
		"NodeBackup","testgres_config",
		"TestgresConfig","configure_testgres","scoped_config","push_config","pop_config",
		"NodeConnection","DatabaseError","InternalError","ProgrammingError","OperationalError",
		"TestgresException","ExecUtilException","QueryException","TimeoutException","CatchUpException","StartNodeException","InitNodeException","BackupException",
		"TestgresException","ExecUtilException","QueryException","TimeoutException","CatchUpException","StartNodeException","InitNodeException","BackupException","InvalidOperationException",
		"XLogMethod","IsolationLevel","NodeStatus","ProcessType","DumpFormat",
		"PostgresNode","NodeApp",
		"reserve_port","release_port","bound_ports","get_bin_path","get_pg_config","get_pg_version",
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,13 +9,14 @@ class TestgresException(Exception):

		@six.python_2_unicode_compatible
		classExecUtilException(TestgresException):
		def__init__(self,message=None,command=None,exit_code=0,out=None):
		def__init__(self,message=None,command=None,exit_code=0,out=None,error=None):
		super(ExecUtilException,self).__init__(message)

		self.message=message
		self.command=command
		self.exit_code=exit_code
		self.out=out
		self.error=error

		def__str__(self):
		msg= []
Expand All		@@ -24,13 +25,17 @@ def __str__(self):
		msg.append(self.message)

		ifself.command:
		msg.append(u'Command: {}'.format(self.command))
		command_s=' '.join(self.command)ifisinstance(self.command,list)elseself.command,
		msg.append(u'Command: {}'.format(command_s))

		ifself.exit_code:
		msg.append(u'Exit code: {}'.format(self.exit_code))

		ifself.error:
		msg.append(u'---- Error:\n{}'.format(self.error))

		ifself.out:
		msg.append(u'----\n{}'.format(self.out))
		msg.append(u'---- Out:\n{}'.format(self.out))

		returnself.convert_and_join(msg)

Expand DownExpand Up		@@ -98,3 +103,7 @@ class InitNodeException(TestgresException):

		classBackupException(TestgresException):
		pass


		classInvalidOperationException(TestgresException):
		pass
dmitry-lipetsk marked this conversation as resolved. Show resolvedHide resolved
Original file line number	Diff line number	Diff line change
Expand Up		@@ -74,7 +74,8 @@
		TimeoutException, \
		InitNodeException, \
		TestgresException, \
		BackupException
		BackupException, \
		InvalidOperationException

		from .logger import TestgresLogger

Expand DownExpand Up		@@ -987,6 +988,37 @@ def psql(self,
		>>> psql(query='select 3', ON_ERROR_STOP=1)
		"""

		return self._psql(
		ignore_errors=True,
		query=query,
		filename=filename,
		dbname=dbname,
		username=username,
		input=input,
		**variables
		)

		def _psql(
		self,
		ignore_errors,
		query=None,
		filename=None,
		dbname=None,
		username=None,
		input=None,
		**variables):
		assert type(variables) == dict # noqa: E721

		#
		# We do not support encoding. It may be added later. Ok?
		#
		if input is None:
		pass
		elif type(input) == bytes: # noqa: E721
		pass
		else:
		raise Exception("Input data must be None or bytes.")

		dbname = dbname or default_dbname()

		psql_params = [
Expand DownExpand Up		@@ -1017,20 +1049,14 @@ def psql(self,

		# should be the last one
		psql_params.append(dbname)
		if not self.os_ops.remote:
		# start psql process
		process = subprocess.Popen(psql_params,
		stdin=subprocess.PIPE,
		stdout=subprocess.PIPE,
		stderr=subprocess.PIPE)

		# wait until it finishes and get stdout and stderr
		out, err = process.communicate(input=input)
		return process.returncode, out, err
		else:
		status_code, out, err = self.os_ops.exec_command(psql_params, verbose=True, input=input)

		return status_code, out, err
		return self.os_ops.exec_command(
		psql_params,
		verbose=True,
		input=input,
		stderr=subprocess.PIPE,
		stdout=subprocess.PIPE,
		ignore_errors=ignore_errors)

		@method_decorator(positional_args_hack(['dbname', 'query']))
		def safe_psql(self, query=None, expect_error=False, **kwargs):
Expand All		@@ -1051,22 +1077,27 @@ def safe_psql(self, query=None, expect_error=False, **kwargs):
		Returns:
		psql's output as str.
		"""
		assert type(kwargs) == dict # noqa: E721
		assert not ("ignore_errors" in kwargs.keys())
		assert not ("expect_error" in kwargs.keys())

		# force this setting
		kwargs['ON_ERROR_STOP'] = 1
		try:
		ret, out, err = self.psql(query=query, **kwargs)
		ret, out, err = self._psql(ignore_errors=False,query=query, **kwargs)
		except ExecUtilException as e:
		ret = e.exit_code
		out = e.out
		err = e.message
		if ret:
		if expect_error:
		out = (err or b'').decode('utf-8')
		else:
		raise QueryException((err or b'').decode('utf-8'), query)
		elif expect_error:
		assert False, "Exception was expected, but query finished successfully: `{}` ".format(query)
		if not expect_error:
		raise QueryException(e.message, query)

		if type(e.error) == bytes: # noqa: E721
		return e.error.decode("utf-8") # throw

		# [2024-12-09] This situation is not expected
		assert False
		return e.error

		if expect_error:
		raise InvalidOperationException("Exception was expected, but query finished successfully: `{}`.".format(query))

		return out

Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,52 @@
		importlocale


		classHelpers:
		def_make_get_default_encoding_func():
		# locale.getencoding is added in Python 3.11
		ifhasattr(locale,'getencoding'):
		returnlocale.getencoding

		# It must exist
		returnlocale.getpreferredencoding

		# Prepared pointer on function to get a name of system codepage
		_get_default_encoding_func=_make_get_default_encoding_func()

		defGetDefaultEncoding():
		#
		# Original idea/source was:
		#
		# def os_ops.get_default_encoding():
		# if not hasattr(locale, 'getencoding'):
		# locale.getencoding = locale.getpreferredencoding
		# return locale.getencoding() or 'UTF-8'
		#

		assert__class__._get_default_encoding_funcisnotNone

		r=__class__._get_default_encoding_func()

		ifr:
		assertrisnotNone
		asserttype(r)==str# noqa: E721
		assertr!=""
		returnr

		# Is it an unexpected situation?
		return'UTF-8'

		defPrepareProcessInput(input,encoding):
		ifnotinput:
		returnNone

		iftype(input)==str:# noqa: E721
		ifencodingisNone:
		returninput.encode(__class__.GetDefaultEncoding())

		asserttype(encoding)==str# noqa: E721
		returninput.encode(encoding)

		# It is expected!
		asserttype(input)==bytes# noqa: E721
		returninput
Original file line number	Diff line number	Diff line change
Expand Up		@@ -11,6 +11,8 @@

		from ..exceptionsimportExecUtilException
		from .os_opsimportConnectionParams,OsOperations,pglib,get_default_encoding
		from .raise_errorimportRaiseError
		from .helpersimportHelpers

		try:
		fromshutilimportwhichasfind_executable
Expand DownExpand Up		@@ -47,14 +49,6 @@ def __init__(self, conn_params=None):
		self.remote=False
		self.username=conn_params.usernameorgetpass.getuser()

		@staticmethod
		def_raise_exec_exception(message,command,exit_code,output):
		"""Raise an ExecUtilException."""
		raiseExecUtilException(message=message.format(output),
		command=' '.join(command)ifisinstance(command,list)elsecommand,
		exit_code=exit_code,
		out=output)

		@staticmethod
		def_process_output(encoding,temp_file_path):
		"""Process the output of a command from a temporary file."""
Expand All		@@ -65,6 +59,8 @@ def _process_output(encoding, temp_file_path):
		returnoutput,None# In Windows stderr writing in stdout

		def_run_command__nt(self,cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding):
		# TODO: why don't we use the data from input?

		withtempfile.NamedTemporaryFile(mode='w+b',delete=False)astemp_file:
		stdout=temp_file
		stderr=subprocess.STDOUT
Expand All		@@ -86,25 +82,36 @@ def _run_command__nt(self, cmd, shell, input, stdin, stdout, stderr, get_process
		returnprocess,output,error

		def_run_command__generic(self,cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding):
		input_prepared=None
		ifnotget_process:
		input_prepared=Helpers.PrepareProcessInput(input,encoding)# throw

		assertinput_preparedisNoneor (type(input_prepared)==bytes)# noqa: E721

		process=subprocess.Popen(
		cmd,
		shell=shell,
		stdin=stdinorsubprocess.PIPEifinputisnotNoneelseNone,
		stdout=stdoutorsubprocess.PIPE,
		stderr=stderrorsubprocess.PIPE,
		)
		assertnot (processisNone)
		ifget_process:
		returnprocess,None,None
		try:
		output,error=process.communicate(input=input.encode(encoding)ifinputelseNone,timeout=timeout)
		ifencoding:
		output=output.decode(encoding)
		error=error.decode(encoding)
		returnprocess,output,error
		output,error=process.communicate(input=input_prepared,timeout=timeout)
		exceptsubprocess.TimeoutExpired:
		process.kill()
		raiseExecUtilException("Command timed out after {} seconds.".format(timeout))

		asserttype(output)==bytes# noqa: E721
		asserttype(error)==bytes# noqa: E721

		ifencoding:
		output=output.decode(encoding)
		error=error.decode(encoding)
		returnprocess,output,error

		def_run_command(self,cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding):
		"""Execute a command and return the process and its output."""
		ifos.name=='nt'andstdoutisNone:# Windows
Expand All		@@ -120,11 +127,20 @@ def exec_command(self, cmd, wait_exit=False, verbose=False, expect_error=False,
		"""
		Execute a command in a subprocess and handle the output based on the provided parameters.
		"""
		asserttype(expect_error)==bool# noqa: E721
		asserttype(ignore_errors)==bool# noqa: E721

		process,output,error=self._run_command(cmd,shell,input,stdin,stdout,stderr,get_process,timeout,encoding)
		ifget_process:
		returnprocess
		ifnotignore_errorsand ((process.returncode!=0orhas_errors(output=output,error=error))andnotexpect_error):
		self._raise_exec_exception('Utility exited with non-zero code. Error `{}`',cmd,process.returncode,errororoutput)
		RaiseError.UtilityExitedWithNonZeroCode(
		cmd=cmd,
		exit_code=process.returncode,
		msg_arg=errororoutput,
		error=error,
		out=output
		)

		ifverbose:
		returnprocess.returncode,output,error
Expand Down
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,46 @@
		from ..exceptions import ExecUtilException
		from .helpers import Helpers


		class RaiseError:
		def UtilityExitedWithNonZeroCode(cmd, exit_code, msg_arg, error, out):
		assert type(exit_code) == int # noqa: E721

		msg_arg_s = __class__._TranslateDataIntoString(msg_arg).strip()
		assert type(msg_arg_s) == str # noqa: E721

		if msg_arg_s == "":
		msg_arg_s = "#no_error_message"

		message = "Utility exited with non-zero code. Error: `" + msg_arg_s + "`"
		raise ExecUtilException(
		message=message,
		command=cmd,
		exit_code=exit_code,
		out=out,
		error=error)

		def _TranslateDataIntoString(data):
		if type(data) == bytes: # noqa: E721
		return __class__._TranslateDataIntoString__FromBinary(data)

		return str(data)

		def _TranslateDataIntoString__FromBinary(data):
		assert type(data) == bytes # noqa: E721

		try:
		return data.decode(Helpers.GetDefaultEncoding())
		except UnicodeDecodeError:
		pass

		return "#cannot_decode_text"

		def _BinaryIsASCII(data):
		assert type(data) == bytes # noqa: E721

		for b in data:
		if not (b >= 0 and b <= 127):
		return False

		return True