| #!/usr/bin/env python3 |
| |
| # Copyright 2017 The Chromium Authors |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Runs a gtest-based test on Swarming, optionally many times, collecting the |
| output of the runs into a directory. Useful for flake checking, and faster than |
| using trybots by avoiding repeated bot_update, compile, archive, etc. and |
| allowing greater parallelism. |
| |
| To use, run in a new shell (it blocks until all Swarming jobs complete): |
| |
| tools/run-swarmed.py out/rel base_unittests |
| |
| The logs of the runs will be stored in results/ (or specify a results directory |
| with --results=some_dir). You can then do something like `grep -L SUCCESS |
| results/*` to find the tests that failed or otherwise process the log files. |
| |
| See //docs/workflow/debugging-with-swarming.md for more details. |
| """ |
| |
| |
| |
| import argparse |
| import hashlib |
| import json |
| import multiprocessing.dummy |
| import os |
| import shutil |
| import subprocess |
| import sys |
| import traceback |
| |
| CHROMIUM_ROOT= os.path.join(os.path.dirname(__file__), os.pardir) |
| BUILD_DIR= os.path.join(CHROMIUM_ROOT,'build') |
| |
| if BUILD_DIRnotin sys.path: |
| sys.path.insert(0, BUILD_DIR) |
| import gn_helpers |
| |
| INTERNAL_ERROR_EXIT_CODE=-1000 |
| |
| DEFAULT_ANDROID_DEVICE_TYPE="walleye" |
| |
| |
| def_Spawn(args): |
| """Triggers a swarming job. The arguments passed are: |
| - The index of the job; |
| - The command line arguments object; |
| - The digest of test files. |
| |
| The return value is passed to a collect-style map() and consists of: |
| - The index of the job; |
| - The json file created by triggering and used to collect results; |
| - The command line arguments object. |
| """ |
| try: |
| return_DoSpawn(args) |
| exceptExceptionas e: |
| traceback.print_exc() |
| returnNone |
| |
| |
| def_DoSpawn(args): |
| index, args, cas_digest, swarming_command= args |
| runner_args=[] |
| json_file= os.path.join(args.results,'%d.json'% index) |
| trigger_args=[ |
| 'tools/luci-go/swarming', |
| 'trigger', |
| '-S', |
| f'https://{args.swarming_instance}.appspot.com', |
| '-digest', |
| cas_digest, |
| '-dump-json', |
| json_file, |
| '-tag=purpose:user-debug-run-swarmed', |
| # 30 is try level. So use the same here. |
| '-priority', |
| '30', |
| ] |
| if args.target_os=='fuchsia': |
| trigger_args+=[ |
| '-d', |
| 'kvm=1', |
| ] |
| if args.gpuisNone: |
| trigger_args+=[ |
| '-d', |
| 'gpu=none', |
| ] |
| elif args.target_os=='android': |
| if args.arch=='x86': |
| # No x86 Android devices are available in swarming. So assume we want to |
| # run on emulators when building for x86 on Android. |
| args.swarming_os='Linux' |
| args.pool='chromium.tests.avd' |
| # android_28_google_apis_x86 == Android P emulator. |
| # See //tools/android/avd/proto/ for other options. |
| runner_args.append( |
| '--avd-config=../../tools/android/avd/proto/android_28_google_apis_x86.textpb' |
| ) |
| elif args.device_typeisNoneand args.device_osisNone: |
| # The aliases for device type are stored here: |
| # luci/appengine/swarming/ui2/modules/alias.js |
| # for example 'blueline' = 'Pixel 3' |
| trigger_args+=['-d','device_type='+ DEFAULT_ANDROID_DEVICE_TYPE] |
| elif args.target_os=='ios': |
| runner_args.append(f'--xcode-build-version={args.ios_xcode_build_version}') |
| runner_args.append('--xctest') |
| runner_args.append('--out-dir=${ISOLATED_OUTDIR}') |
| |
| if args.ios_sim_versionand args.ios_sim_platform: |
| # simulator test runner and trigger args |
| runner_args.append(f'--version={args.ios_sim_version}') |
| runner_args.extend(['--platform', args.ios_sim_platform]) |
| |
| version_with_underscore= args.ios_sim_version.replace('.','_') |
| trigger_args.extend([ |
| '-named-cache', f'runtime_ios_{version_with_underscore}' |
| f'=Runtime-ios-{args.ios_sim_version}' |
| ]) |
| elif args.ios_device: |
| # device trigger args |
| trigger_args.extend(['-d', f'device={args.ios_device}']) |
| trigger_args.extend(['-d','device_status=available']) |
| else: |
| raiseException('Either both of --ios-sim-version and --ios-sim-platform ' |
| 'or --ios-device is required') |
| |
| trigger_args.extend( |
| ['-named-cache', f'xcode_ios_{args.ios_xcode_build_version}=Xcode.app']) |
| trigger_args.extend( |
| ['-cipd-package','.:infra/tools/mac_toolchain/${platform}=latest']) |
| |
| if args.service_account: |
| account= args.service_account |
| elif args.swarming_instance=='chromium-swarm': |
| account='chromium-tester@chops-service-accounts.iam.gserviceaccount.com' |
| elif args.swarming_instance=='chrome-swarming': |
| account='chrome-tester@chops-service-accounts.iam.gserviceaccount.com' |
| trigger_args.extend(['-service-account', account]) |
| |
| if args.arch!='detect': |
| trigger_args+=[ |
| '-d', |
| 'cpu='+ args.arch, |
| ] |
| |
| if args.device_type: |
| trigger_args+=['-d','device_type='+ args.device_type] |
| |
| if args.device_os: |
| trigger_args+=['-d','device_os='+ args.device_os] |
| |
| if args.gpu: |
| trigger_args+=['-d','gpu='+ args.gpu] |
| |
| ifnot args.no_test_flags: |
| # These flags are recognized by our test runners, but do not work |
| # when running custom scripts. |
| runner_args+=[ |
| '--test-launcher-summary-output=${ISOLATED_OUTDIR}/output.json' |
| ] |
| if'junit'notin args.target_name: |
| runner_args+=['--system-log-file=${ISOLATED_OUTDIR}/system_log'] |
| if args.gtest_filter: |
| runner_args.append('--gtest_filter='+ args.gtest_filter) |
| if args.gtest_repeat: |
| runner_args.append('--gtest_repeat='+ args.gtest_repeat) |
| if args.test_launcher_shard_indexand args.test_launcher_total_shards: |
| runner_args.append('--test-launcher-shard-index='+ |
| args.test_launcher_shard_index) |
| runner_args.append('--test-launcher-total-shards='+ |
| args.test_launcher_total_shards) |
| elif args.target_os=='fuchsia': |
| filter_file= \ |
| 'testing/buildbot/filters/fuchsia.'+ args.target_name+'.filter' |
| if os.path.isfile(filter_file): |
| runner_args.append('--test-launcher-filter-file=../../'+ filter_file) |
| |
| runner_args.extend(args.runner_args) |
| |
| trigger_args.extend(['-d','os='+ args.swarming_os]) |
| trigger_args.extend(['-d','pool='+ args.pool]) |
| trigger_args.extend(['--relative-cwd', args.out_dir,'--']) |
| trigger_args.extend(swarming_command) |
| trigger_args.extend(runner_args) |
| |
| with open(os.devnull,'w')as nul: |
| subprocess.check_call(trigger_args, stdout=nul) |
| return(index, json_file, args) |
| |
| |
| def_Collect(spawn_result): |
| if spawn_resultisNone: |
| return1 |
| |
| index, json_file, args= spawn_result |
| with open(json_file)as f: |
| task_json= json.load(f) |
| task_ids=[task['task_id']for taskin task_json['tasks']] |
| |
| for tin task_ids: |
| print('Task {}: https://{}.appspot.com/task?id={}'.format( |
| index, args.swarming_instance, t)) |
| p= subprocess.Popen([ |
| 'tools/luci-go/swarming', |
| 'collect', |
| '-S', |
| f'https://{args.swarming_instance}.appspot.com', |
| '--task-output-stdout=console', |
| ]+ task_ids, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.STDOUT) |
| stdout= p.communicate()[0] |
| if p.returncode!=0and len(stdout)<2**10and'Internal error!'in stdout: |
| exit_code= INTERNAL_ERROR_EXIT_CODE |
| file_suffix='.INTERNAL_ERROR' |
| else: |
| exit_code= p.returncode |
| file_suffix=''if exit_code==0else'.FAILED' |
| filename='%d%s.stdout.txt'%(index, file_suffix) |
| with open(os.path.join(args.results, filename),'wb')as f: |
| f.write(stdout) |
| return exit_code |
| |
| |
| def main(): |
| parser= argparse.ArgumentParser() |
| parser.add_argument( |
| '--swarming-instance', |
| choices=['chromium-swarm','chrome-swarming'], |
| default='chromium-swarm', |
| help='The swarming instance where the task(s) will be run.') |
| parser.add_argument('--swarming-os', help='OS specifier for Swarming.') |
| parser.add_argument('--target-os', default='detect', help='gn target_os') |
| parser.add_argument('--arch','-a', default='detect', |
| help='CPU architecture of the test binary.') |
| parser.add_argument('--build', |
| dest='build', |
| action='store_true', |
| help='Build before isolating.') |
| parser.add_argument('--no-build', |
| dest='build', |
| action='store_false', |
| help='Do not build, just isolate (default).') |
| parser.add_argument('--isolate-map-file','-i', |
| help='path to isolate map file if not using default') |
| parser.add_argument('--copies','-n', type=int, default=1, |
| help='Number of copies to spawn.') |
| parser.add_argument( |
| '--device-os', help='Run tests on the given version of Android.') |
| parser.add_argument('--device-type', |
| help='device_type specifier for Swarming' |
| ' from https://chromium-swarm.appspot.com/botlist .') |
| parser.add_argument('--gpu', |
| help='gpu specifier for Swarming' |
| ' from https://chromium-swarm.appspot.com/botlist .') |
| parser.add_argument('--pool', |
| default='chromium.tests', |
| help='Use the given swarming pool.') |
| parser.add_argument('--results','-r', default='results', |
| help='Directory in which to store results.') |
| parser.add_argument( |
| '--gtest_filter', |
| help='Deprecated. Pass as test runner arg instead, like \'-- ' |
| '--gtest_filter="*#testFoo"\'') |
| parser.add_argument( |
| '--gtest_repeat', |
| help='Deprecated. Pass as test runner arg instead, like \'-- ' |
| '--gtest_repeat=99\'') |
| parser.add_argument( |
| '--test-launcher-shard-index', |
| help='Shard index to run. Use with --test-launcher-total-shards.') |
| parser.add_argument('--test-launcher-total-shards', |
| help='Number of shards to split the test into. Use with' |
| ' --test-launcher-shard-index.') |
| parser.add_argument('--no-test-flags', action='store_true', |
| help='Do not add --test-launcher-summary-output and ' |
| '--system-log-file flags to the comment.') |
| parser.add_argument('out_dir', type=str, help='Build directory.') |
| parser.add_argument('target_name', type=str, help='Name of target to run.') |
| parser.add_argument( |
| '--service-account', |
| help='Optional service account that the swarming task will be run using. ' |
| 'Default value will be set based on the "--swarming-instance".') |
| # ios only args |
| parser.add_argument('--ios-xcode-build-version', |
| help='The version of xcode that will be used for all ' |
| 'xcodebuild CLI commands') |
| parser.add_argument('--ios-sim-version', |
| help='iOS simulator version, ex. 17.2') |
| parser.add_argument('--ios-sim-platform', |
| help='iOS simulator platform, ex. iPhone 14') |
| parser.add_argument('--ios-device', |
| help='iOS physical device type, ex. iPhone12,1') |
| parser.add_argument( |
| 'runner_args', |
| nargs='*', |
| type=str, |
| help='Arguments to pass to the test runner, e.g. gtest_filter and ' |
| 'gtest_repeat.') |
| parser.add_argument('--force', |
| action='store_true', |
| help='Bypasses deprecation notice.') |
| |
| args= parser.parse_intermixed_args() |
| |
| # TODO(crbug.com/386167803): Remove this script after this deprecation notice |
| # has been live for a few months. |
| ifnot args.force: |
| print( |
| 'This script is deprecated in favor of the UTR. For more info, see ' |
| 'https://chromium.googlesource.com/chromium/src/+/main/tools/utr/README.md. ' |
| 'To skip this warning, re-run this script with "--force". Note that ' |
| 'this script will be deleted sometime in 2025.', |
| file=sys.stderr) |
| return1 |
| |
| with open(os.path.join(args.out_dir,'args.gn'))as f: |
| gn_args= gn_helpers.FromGNArgs(f.read()) |
| |
| if args.target_os=='detect': |
| if'target_os'in gn_args: |
| args.target_os= gn_args['target_os'].strip('"') |
| else: |
| args.target_os={ |
| 'darwin':'mac', |
| 'linux':'linux', |
| 'win32':'win' |
| }[sys.platform] |
| |
| if args.swarming_osisNone: |
| args.swarming_os={ |
| 'mac':'Mac', |
| 'ios':'Mac', |
| 'win':'Windows', |
| 'linux':'Linux', |
| 'android':'Android', |
| 'fuchsia':'Linux' |
| }[args.target_os] |
| |
| if args.target_os=='win'and args.target_name.endswith('.exe'): |
| # The machinery expects not to have a '.exe' suffix. |
| args.target_name= os.path.splitext(args.target_name)[0] |
| |
| # Determine the CPU architecture of the test binary, if not specified. |
| if args.arch=='detect': |
| if args.target_os=='ios': |
| print('iOS must specify --arch. Probably arm64 or x86-64.') |
| return1 |
| if args.target_osnotin('android','mac','win'): |
| executable_info= subprocess.check_output( |
| ['file', os.path.join(args.out_dir, args.target_name)], text=True) |
| if'ARM aarch64'in executable_info: |
| args.arch='arm64', |
| else: |
| args.arch='x86-64' |
| elif args.target_os=='android': |
| args.arch= gn_args.get('target_cpu','detect') |
| |
| mb_cmd=[sys.executable,'tools/mb/mb.py','isolate'] |
| ifnot args.build: |
| mb_cmd.append('--no-build') |
| if args.isolate_map_file: |
| mb_cmd+=['--isolate-map-file', args.isolate_map_file] |
| mb_cmd+=['//'+ args.out_dir, args.target_name] |
| subprocess.check_call(mb_cmd, shell=os.name=='nt') |
| |
| print('If you get authentication errors, follow:') |
| print( |
| ' https://chromium.googlesource.com/chromium/src/+/HEAD/docs/workflow/debugging-with-swarming.md#authenticating' |
| ) |
| |
| print('Uploading to isolate server, this can take a while...') |
| isolate= os.path.join(args.out_dir, args.target_name+'.isolate') |
| archive_json= os.path.join(args.out_dir, args.target_name+'.archive.json') |
| subprocess.check_output([ |
| 'tools/luci-go/isolate','archive','-cas-instance', |
| args.swarming_instance,'-isolate', isolate,'-dump-json', archive_json |
| ]) |
| with open(archive_json)as f: |
| cas_digest= json.load(f).get(args.target_name) |
| |
| mb_cmd=[ |
| sys.executable,'tools/mb/mb.py','get-swarming-command','--as-list' |
| ] |
| ifnot args.build: |
| mb_cmd.append('--no-build') |
| if args.isolate_map_file: |
| mb_cmd+=['--isolate-map-file', args.isolate_map_file] |
| mb_cmd+=['//'+ args.out_dir, args.target_name] |
| mb_output= subprocess.check_output(mb_cmd, shell=os.name=='nt') |
| swarming_cmd= json.loads(mb_output) |
| |
| if os.path.isdir(args.results): |
| shutil.rmtree(args.results) |
| os.makedirs(args.results) |
| |
| try: |
| print('Triggering %d tasks...'% args.copies) |
| # Use dummy since threadpools give better exception messages |
| # than process pools do, and threads work fine for what we're doing. |
| pool= multiprocessing.dummy.Pool() |
| spawn_args=[(i, args, cas_digest, swarming_cmd) |
| for iin range(args.copies)] |
| spawn_results= pool.imap_unordered(_Spawn, spawn_args) |
| |
| exit_codes=[] |
| collect_results= pool.imap_unordered(_Collect, spawn_results) |
| for resultin collect_results: |
| exit_codes.append(result) |
| successes= sum(1for xin exit_codesif x==0) |
| errors= sum(1for xin exit_codesif x== INTERNAL_ERROR_EXIT_CODE) |
| failures= len(exit_codes)- successes- errors |
| clear_to_eol='\033[K' |
| print( |
| '\r[%d/%d] collected: ' |
| '%d successes, %d failures, %d bot errors...%s'% |
| (len(exit_codes), args.copies, successes, failures, errors, |
| clear_to_eol), |
| end=' ') |
| sys.stdout.flush() |
| |
| print() |
| print('Results logs collected into', os.path.abspath(args.results)+'.') |
| finally: |
| pool.close() |
| pool.join() |
| return0 |
| |
| |
| if __name__=='__main__': |
| sys.exit(main()) |
