Jun 11, 2022 · Jun 10, 2022 · Jun 10, 2022 · Jun 10, 2022
diff --git a/Misc/NEWS.d/next/Tests/2022-06-10-21-18-14.gh-issue-84461.9TAb26.rst b/Misc/NEWS.d/next/Tests/2022-06-10-21-18-14.gh-issue-84461.9TAb26.rst
 ``run_tests.py`` now handles cross compiling env vars correctly and pass
 ``HOSTRUNNER`` to regression tests.
diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py
 """

 import os
 import shlex
 import sys
 import sysconfig
 import test.support


 def is_resource_use_flag(arg):
    return arg.startswith('-u') or arg.startswith('--use')

 def is_python_flag(arg):
    return arg.startswith('-p') or arg.startswith('--python')


 def main(regrtest_args):
    args = [sys.executable,
            '-u',                 # Unbuffered stdout and stderr
            '-W', 'default',      # Warnings set to 'default'
            '-bb',                # Warnings about bytes/bytearray
            '-E',                 # Ignore environment variables
            ]

    cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
    hostrunner = os.environ.get("_PYTHON_HOSTRUNNER")
    if hostrunner is None:
        hostrunner = sysconfig.get_config_var("HOSTRUNNER")
    if cross_compile:
        # emulate -E, but keep PYTHONPATH + cross compile env vars, so
        # test executable can load correct sysconfigdata file.
        keep = {
            '_PYTHON_PROJECT_BASE',
            '_PYTHON_HOST_PLATFORM',
            '_PYTHON_SYSCONFIGDATA_NAME',
            'PYTHONPATH'
        }
        environ = {
            name: value for name, value in os.environ.items()
            if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
        }
    else:
        environ = os.environ.copy()
        args.append("-E")

    # Allow user-specified interpreter options to override our defaults.
    args.extend(test.support.args_from_interpreter_flags())

    if sys.platform == 'win32':
        args.append('-n')         # Silence alerts under Windows
    if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
        args.extend(['-j', '0'])  # Use all CPU cores
        if cross_compile and hostrunner:
            # For now use only one core for cross compiled builds.
            # hostrunner can be expensive.
            args.extend(['-j', '1'])
        else:
            args.extend(['-j', '0'])  # Use all CPU cores
    if not any(is_resource_use_flag(arg) for arg in regrtest_args):
        args.extend(['-u', 'all,-largefile,-audio,-gui'])

    if cross_compile and hostrunner:
        # If HOSTRUNNER is set and -p/--python option is not given, then
        # use hostrunner to execute python binary for tests.
        if not any(is_python_flag(arg) for arg in regrtest_args):
            buildpython = sysconfig.get_config_var("BUILDPYTHON")
            args.extend(["--python", f"{hostrunner} {buildpython}"])

    args.extend(regrtest_args)
    print(' '.join(args))

    print(shlex.join(args))
    if sys.platform == 'win32':
        from subprocess import call
        sys.exit(call(args))
    else:
        os.execv(sys.executable, args)
        os.execve(sys.executable, args, environ)


 if __name__ == '__main__':
diff --git a/configure b/configure
diff --git a/configure.ac b/configure.ac
    dnl TODO: support other WASI runtimes
    dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the
    dnl directory containing _sysconfigdata to PYTHONPATH.
    [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=$$(realpath --relative-to $(abs_srcdir) $(abs_builddir))/$$(cat pybuilddir.txt) --mapdir /::$(srcdir) --'],
    [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shellrealpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shellcat pybuilddir.txt) --mapdir /::$(srcdir) --'],
    [HOSTRUNNER='']
  )
 fi
 AC_SUBST([HOSTRUNNER])
 AC_MSG_RESULT([$HOSTRUNNER])

 if test -n "$HOSTRUNNER"; then
  dnl Pass hostrunner variable as env var in order to expand shell expressions.
  PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD"
 fi

 AC_MSG_RESULT($LDLIBRARY)

 # LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		``run_tests.py`` now handles cross compiling env vars correctly and pass
		``HOSTRUNNER`` to regression tests.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -8,7 +8,9 @@
		"""

		import os
		import shlex
		import sys
		import sysconfig
		import test.support


Expand All		@@ -19,15 +21,38 @@ def is_multiprocess_flag(arg):
		def is_resource_use_flag(arg):
		return arg.startswith('-u') or arg.startswith('--use')

		def is_python_flag(arg):
		return arg.startswith('-p') or arg.startswith('--python')


		def main(regrtest_args):
		args = [sys.executable,
		'-u', # Unbuffered stdout and stderr
		'-W', 'default', # Warnings set to 'default'
		'-bb', # Warnings about bytes/bytearray
		'-E', # Ignore environment variables
		]

		cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
		hostrunner = os.environ.get("_PYTHON_HOSTRUNNER")
		if hostrunner is None:
tiran marked this conversation as resolved. OutdatedShow resolvedHide resolved
		hostrunner = sysconfig.get_config_var("HOSTRUNNER")
		if cross_compile:
		# emulate -E, but keep PYTHONPATH + cross compile env vars, so
		# test executable can load correct sysconfigdata file.
		keep = {
		'_PYTHON_PROJECT_BASE',
		'_PYTHON_HOST_PLATFORM',
		'_PYTHON_SYSCONFIGDATA_NAME',
		'PYTHONPATH'
		}
		environ = {
		name: value for name, value in os.environ.items()
		if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
		}
		else:
		environ = os.environ.copy()
		args.append("-E")

		# Allow user-specified interpreter options to override our defaults.
		args.extend(test.support.args_from_interpreter_flags())

Expand All		@@ -38,16 +63,30 @@ def main(regrtest_args):
		if sys.platform == 'win32':
		args.append('-n') # Silence alerts under Windows
		if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
		args.extend(['-j', '0']) # Use all CPU cores
		if cross_compile and hostrunner:
		# For now use only one core for cross compiled builds.
		# hostrunner can be expensive.
tiran marked this conversation as resolved. OutdatedShow resolvedHide resolved
		args.extend(['-j', '1'])
		else:
		args.extend(['-j', '0']) # Use all CPU cores
		if not any(is_resource_use_flag(arg) for arg in regrtest_args):
		args.extend(['-u', 'all,-largefile,-audio,-gui'])

		if cross_compile and hostrunner:
		# If HOSTRUNNER is set and -p/--python option is not given, then
		# use hostrunner to execute python binary for tests.
		if not any(is_python_flag(arg) for arg in regrtest_args):
		buildpython = sysconfig.get_config_var("BUILDPYTHON")
		args.extend(["--python", f"{hostrunner} {buildpython}"])

		args.extend(regrtest_args)
		print(' '.join(args))

		print(shlex.join(args))
		if sys.platform == 'win32':
		from subprocess import call
		sys.exit(call(args))
		else:
		os.execv(sys.executable, args)
		os.execve(sys.executable, args, environ)


		if __name__ == '__main__':
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -1492,13 +1492,18 @@ then
		dnl TODO: support other WASI runtimes
		dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the
		dnl directory containing _sysconfigdata to PYTHONPATH.
		[WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=$$(realpath --relative-to $(abs_srcdir) $(abs_builddir))/$$(cat pybuilddir.txt) --mapdir /::$(srcdir) --'],
		[WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shellrealpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shellcat pybuilddir.txt) --mapdir /::$(srcdir) --'],
		[HOSTRUNNER='']
		)
		fi
		AC_SUBST([HOSTRUNNER])
		AC_MSG_RESULT([$HOSTRUNNER])

		if test -n "$HOSTRUNNER"; then
		dnl Pass hostrunner variable as env var in order to expand shell expressions.
		PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD"
		fi

		AC_MSG_RESULT($LDLIBRARY)

		# LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable
Expand Down