Uh oh!
There was an error while loading.Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork32k
gh-131531: android.py enhancements to support cibuildwheel#132870
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Open
mhsmith wants to merge10 commits intopython:mainChoose a base branch
base:main
Could not load branches
Branch not found:{{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline, and old review comments may become outdated.
+237 −85
Open
Changes fromall commits
Commits
Show all changes
10 commits Select commitHold shift + click to select a range
71cccb3 Add `android.py env` command
mhsmithe3d27ac Miscellaneous cleanups
mhsmith6156255 Merge branch 'main' into cibuildwheel
mhsmith24b082f Prefer 'encoding' to 'text'
mhsmithb7461d3 Clarify sysconfigdata file discovery
mhsmithc7cdb98 Rename api_level environment variable to ANDROID_API_LEVEL
mhsmithf29e177 Environment variable cleanups
mhsmithfc8c1e1 Add `android.py test` -c and -m options
mhsmithb273bc7 Add `android.py test` --site-packages and --cwd options
mhsmith9c46ab0 Merge remote-tracking branch 'origin/main' into cibuildwheel
mhsmithFile filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -14,17 +14,21 @@ | ||
| from contextlib import asynccontextmanager | ||
| from datetime import datetime, timezone | ||
| from glob import glob | ||
| from os.path importabspath,basename, relpath | ||
| from pathlib import Path | ||
| from subprocess import CalledProcessError | ||
| from tempfile import TemporaryDirectory | ||
| SCRIPT_NAME = Path(__file__).name | ||
| ANDROID_DIR = Path(__file__).resolve().parent | ||
| PYTHON_DIR = ANDROID_DIR.parent | ||
| in_source_tree = ( | ||
| ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() | ||
| ) | ||
| TESTBED_DIR = ANDROID_DIR / "testbed" | ||
| CROSS_BUILD_DIR =PYTHON_DIR / "cross-build" | ||
| HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] | ||
| APP_ID = "org.python.testbed" | ||
| @@ -74,41 +78,81 @@ def subdir(*parts, create=False): | ||
| def run(command, *, host=None, env=None, log=True, **kwargs): | ||
| kwargs.setdefault("check", True) | ||
| if env is None: | ||
| env = os.environ.copy() | ||
| if host: | ||
| # The -I and -L arguments used when building Python should not be reused | ||
| # when building third-party extension modules, so pass them via the | ||
| # NODIST environment variables. | ||
| host_env = android_env(host) | ||
| for name in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. Why are these needed for the testbed environment? Is this script being used to drive cibuildwheel? Or - I guess - why are they needed now to support compilation when they weren't required previously? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. This section of the script is only directly used when building Python itself. This PR doesn't change the flags, it only moves the flags which contain build-time paths from the This may be unnecessary for cibuildwheel, because it contains code to rewrite any paths in the sysconfigdata. But cibuildwheel isn't the only context in which packages could be built – e.g. we may also see "native" (non-cross) compilation on Termux. | ||
| flags = [] | ||
| nodist = [] | ||
| for word in host_env[name].split(): | ||
| (nodist if word.startswith(("-I", "-L")) else flags).append(word) | ||
| host_env[name] = " ".join(flags) | ||
| host_env[f"{name}_NODIST"] = " ".join(nodist) | ||
| print_env(host_env) | ||
| env.update(host_env) | ||
| if log: | ||
| print(">",join_command(command)) | ||
| return subprocess.run(command, env=env, **kwargs) | ||
| # Format a command so it can be copied into a shell. Like shlex.join, but also | ||
| # accepts arguments which are Paths, or a single string/Path outside of a list. | ||
| def join_command(args): | ||
| if isinstance(args, (str, Path)): | ||
| return str(args) | ||
| else: | ||
| return shlex.join(map(str, args)) | ||
| # Format the environment so it can be pasted into a shell. | ||
| def print_env(env): | ||
| for key, value in sorted(env.items()): | ||
| print(f"export {key}={shlex.quote(value)}") | ||
| def android_env(host): | ||
| if host: | ||
| prefix = subdir(host) / "prefix" | ||
| else: | ||
| prefix = ANDROID_DIR / "prefix" | ||
| sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") | ||
| sysconfig_filename = next(sysconfig_files).name | ||
| host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] | ||
| env_script = ANDROID_DIR / "android-env.sh" | ||
| env_output = subprocess.run( | ||
| f"set -eu; " | ||
| f"export HOST={host}; " | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. Is there a reason this has become a full export? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more.
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. I'm also using it in the cibuildwheel PR to set | ||
| f"PREFIX={prefix}; " | ||
| f". {env_script}; " | ||
| f"export", | ||
| check=True, shell=True, capture_output=True, encoding='utf-8', | ||
| ).stdout | ||
| env = {} | ||
| for line in env_output.splitlines(): | ||
| # We don't require every line to match, as there may be some other | ||
| # output from installing the NDK. | ||
| if match := re.search( | ||
| "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line | ||
| ): | ||
| key, value = match[2], match[3] | ||
| if os.environ.get(key) != value: | ||
| env[key] = value | ||
mhsmith marked this conversation as resolved. Show resolvedHide resolvedUh oh!There was an error while loading.Please reload this page. | ||
| if not env: | ||
| raise ValueError(f"Found no variables in {env_script.name} output:\n" | ||
| + env_output) | ||
| return env | ||
| def build_python_path(): | ||
| """The path to the build Python binary.""" | ||
| build_dir = subdir("build") | ||
| @@ -127,7 +171,7 @@ def configure_build_python(context): | ||
| clean("build") | ||
| os.chdir(subdir("build", create=True)) | ||
| command = [relpath(PYTHON_DIR / "configure")] | ||
| if context.args: | ||
| command.extend(context.args) | ||
| run(command) | ||
| @@ -167,7 +211,7 @@ def configure_host_python(context): | ||
| os.chdir(host_dir) | ||
| command = [ | ||
| # Basic cross-compiling configuration | ||
| relpath(PYTHON_DIR / "configure"), | ||
| f"--host={context.host}", | ||
| f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", | ||
| f"--with-build-python={build_python_path()}", | ||
| @@ -196,9 +240,12 @@ def make_host_python(context): | ||
| for pattern in ("include/python*", "lib/libpython*", "lib/python*"): | ||
| delete_glob(f"{prefix_dir}/{pattern}") | ||
| # The Android environment variables were already captured in the Makefile by | ||
| # `configure`, and passing them again when running `make` may cause some | ||
| # flags to be duplicated. So we don't use the `host` argument here. | ||
| os.chdir(host_dir) | ||
| run(["make", "-j", str(os.cpu_count())]) | ||
| run(["make", "install", f"prefix={prefix_dir}"]) | ||
| def build_all(context): | ||
| @@ -474,24 +521,47 @@ async def gradle_task(context): | ||
| task_prefix = "connected" | ||
| env["ANDROID_SERIAL"] = context.connected | ||
| hidden_output = [] | ||
| def log(line): | ||
| # Gradle may take several minutes to install SDK packages, so it's worth | ||
| # showing those messages even in non-verbose mode. | ||
| if context.verbose or line.startswith('Preparing "Install'): | ||
| sys.stdout.write(line) | ||
| else: | ||
| hidden_output.append(line) | ||
| if context.command: | ||
| mode = "-c" | ||
| module = context.command | ||
| else: | ||
| mode = "-m" | ||
| module = context.module or "test" | ||
| args = [ | ||
| gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", | ||
| ] + [ | ||
| # Build-time properties | ||
| f"-Ppython.{name}={value}" | ||
| for name, value in [ | ||
| ("sitePackages", context.site_packages), ("cwd", context.cwd) | ||
| ] if value | ||
| ] + [ | ||
| # Runtime properties | ||
| f"-Pandroid.testInstrumentationRunnerArguments.python{name}={value}" | ||
| for name, value in [ | ||
| ("Mode", mode), ("Module", module), ("Args", join_command(context.args)) | ||
| ] if value | ||
| ] | ||
| log("> " + join_command(args)) | ||
| try: | ||
| async with async_process( | ||
| *args, cwd=TESTBED_DIR, env=env, | ||
| stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | ||
| ) as process: | ||
| while line := (await process.stdout.readline()).decode(*DECODE_ARGS): | ||
| log(line) | ||
| status = await wait_for(process.wait(), timeout=1) | ||
| if status == 0: | ||
| @@ -604,6 +674,10 @@ def package(context): | ||
| print(f"Wrote {package_path}") | ||
| def env(context): | ||
| print_env(android_env(getattr(context, "host", None))) | ||
| # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated | ||
| # by the buildbot worker, we'll make an attempt to clean up our subprocesses. | ||
| def install_signal_handler(): | ||
| @@ -615,36 +689,41 @@ def signal_handler(*args): | ||
| def parse_args(): | ||
| parser = argparse.ArgumentParser() | ||
| subcommands = parser.add_subparsers(dest="subcommand", required=True) | ||
| # Subcommands | ||
| build = subcommands.add_parser( | ||
| "build", help="Run configure-build, make-build, configure-host and " | ||
| "make-host") | ||
| configure_build = subcommands.add_parser( | ||
| "configure-build", help="Run `configure` for the build Python") | ||
| subcommands.add_parser( | ||
| "make-build", help="Run `make` for the build Python") | ||
| configure_host = subcommands.add_parser( | ||
| "configure-host", help="Run `configure` for Android") | ||
| make_host = subcommands.add_parser( | ||
| "make-host", help="Run `make` for Android") | ||
| subcommands.add_parser("clean", help="Delete all build directories") | ||
| subcommands.add_parser("build-testbed", help="Build the testbed app") | ||
| test = subcommands.add_parser("test", help="Run the testbed app") | ||
| package = subcommands.add_parser("package", help="Make a release package") | ||
| env = subcommands.add_parser("env", help="Print environment variables") | ||
| # Common arguments | ||
| for subcommand in build, configure_build, configure_host: | ||
| subcommand.add_argument( | ||
| "--clean", action="store_true", default=False, dest="clean", | ||
| help="Delete the relevant build directories first") | ||
| host_commands = [build, configure_host, make_host, package] | ||
| if in_source_tree: | ||
| host_commands.append(env) | ||
| for subcommand in host_commands: | ||
| subcommand.add_argument( | ||
| "host", metavar="HOST", choices=HOSTS, | ||
| help="Host triplet: choices=[%(choices)s]") | ||
| for subcommand in build, configure_build, configure_host: | ||
| subcommand.add_argument("args", nargs="*", | ||
| help="Extra arguments to pass to `configure`") | ||
| @@ -654,15 +733,32 @@ def parse_args(): | ||
| "-v", "--verbose", action="count", default=0, | ||
| help="Show Gradle output, and non-Python logcat messages. " | ||
| "Use twice to include high-volume messages which are rarely useful.") | ||
| device_group = test.add_mutually_exclusive_group(required=True) | ||
| device_group.add_argument( | ||
| "--connected", metavar="SERIAL", help="Run on a connected device. " | ||
| "Connect it yourself, then get its serial from `adb devices`.") | ||
| device_group.add_argument( | ||
| "--managed", metavar="NAME", help="Run on a Gradle-managed device. " | ||
| "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") | ||
| test.add_argument( | ||
| "--site-packages", metavar="DIR", type=abspath, | ||
| help="Directory to copy as the app's site-packages.") | ||
| test.add_argument( | ||
| "--cwd", metavar="DIR", type=abspath, | ||
| help="Directory to copy as the app's working directory.") | ||
| mode_group = test.add_mutually_exclusive_group() | ||
| mode_group.add_argument( | ||
| "-c", dest="command", help="Execute the given Python code.") | ||
| mode_group.add_argument( | ||
| "-m", dest="module", help="Execute the module with the given name.") | ||
| test.epilog = ( | ||
| "If neither -c nor -m are passed, the default is '-m test', which will " | ||
| "run Python's own test suite.") | ||
| test.add_argument( | ||
| "args", nargs="*", help=f"Arguments to add to sys.argv. " | ||
| f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") | ||
| return parser.parse_args() | ||
| @@ -688,6 +784,7 @@ def main(): | ||
| "build-testbed": build_testbed, | ||
| "test": run_testbed, | ||
| "package": package, | ||
| "env": env, | ||
| } | ||
| try: | ||
| @@ -708,14 +805,9 @@ def print_called_process_error(e): | ||
| if not content.endswith("\n"): | ||
| stream.write("\n") | ||
| # shlex uses single quotes, so we surround the command with double quotes. | ||
| print( | ||
| f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' | ||
| ) | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.