# Copyright 2017 the V8 project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. from collections import OrderedDict import json import optparse import os import sys # Add testrunner to the path. sys.path.insert( 0, os.path.dirname( os.path.dirname(os.path.abspath(__file__)))) from local import testsuite from local import utils from testproc.shard import ShardProc BASE_DIR = ( os.path.dirname( os.path.dirname( os.path.dirname( os.path.abspath(__file__))))) DEFAULT_OUT_GN = 'out.gn' ARCH_GUESS = utils.DefaultArch() # Map of test name synonyms to lists of test suites. Should be ordered by # expected runtimes (suites with slow test cases first). These groups are # invoked in separate steps on the bots. TEST_MAP = { # This needs to stay in sync with test/bot_default.isolate. "bot_default": [ "debugger", "mjsunit", "cctest", "wasm-spec-tests", "inspector", "webkit", "mkgrokdump", "fuzzer", "message", "preparser", "intl", "unittests", ], # This needs to stay in sync with test/default.isolate. "default": [ "debugger", "mjsunit", "cctest", "wasm-spec-tests", "inspector", "mkgrokdump", "fuzzer", "message", "preparser", "intl", "unittests", ], # This needs to stay in sync with test/d8_default.isolate. "d8_default": [ # TODO(machenbach): uncomment after infra side lands. #"debugger", "mjsunit", "webkit", #"message", #"preparser", #"intl", ], # This needs to stay in sync with test/optimize_for_size.isolate. "optimize_for_size": [ "debugger", "mjsunit", "cctest", "inspector", "webkit", "intl", ], "unittests": [ "unittests", ], } class ModeConfig(object): def __init__(self, flags, timeout_scalefactor, status_mode, execution_mode): self.flags = flags self.timeout_scalefactor = timeout_scalefactor self.status_mode = status_mode self.execution_mode = execution_mode DEBUG_FLAGS = ["--nohard-abort", "--enable-slow-asserts", "--verify-heap"] RELEASE_FLAGS = ["--nohard-abort"] MODES = { "debug": ModeConfig( flags=DEBUG_FLAGS, timeout_scalefactor=4, status_mode="debug", execution_mode="debug", ), "optdebug": ModeConfig( flags=DEBUG_FLAGS, timeout_scalefactor=4, status_mode="debug", execution_mode="debug", ), "release": ModeConfig( flags=RELEASE_FLAGS, timeout_scalefactor=1, status_mode="release", execution_mode="release", ), # Normal trybot release configuration. There, dchecks are always on which # implies debug is set. Hence, the status file needs to assume debug-like # behavior/timeouts. "tryrelease": ModeConfig( flags=RELEASE_FLAGS, timeout_scalefactor=1, status_mode="debug", execution_mode="release", ), # This mode requires v8 to be compiled with dchecks and slow dchecks. "slowrelease": ModeConfig( flags=RELEASE_FLAGS + ["--enable-slow-asserts"], timeout_scalefactor=2, status_mode="debug", execution_mode="release", ), } class TestRunnerError(Exception): pass class BuildConfig(object): def __init__(self, build_config): # In V8 land, GN's x86 is called ia32. if build_config['v8_target_cpu'] == 'x86': self.arch = 'ia32' else: self.arch = build_config['v8_target_cpu'] self.is_debug = build_config['is_debug'] self.asan = build_config['is_asan'] self.cfi_vptr = build_config['is_cfi'] self.dcheck_always_on = build_config['dcheck_always_on'] self.gcov_coverage = build_config['is_gcov_coverage'] self.msan = build_config['is_msan'] self.no_i18n = not build_config['v8_enable_i18n_support'] self.no_snap = not build_config['v8_use_snapshot'] self.predictable = build_config['v8_enable_verify_predictable'] self.tsan = build_config['is_tsan'] self.ubsan_vptr = build_config['is_ubsan_vptr'] def __str__(self): detected_options = [] if self.asan: detected_options.append('asan') if self.cfi_vptr: detected_options.append('cfi_vptr') if self.dcheck_always_on: detected_options.append('dcheck_always_on') if self.gcov_coverage: detected_options.append('gcov_coverage') if self.msan: detected_options.append('msan') if self.no_i18n: detected_options.append('no_i18n') if self.no_snap: detected_options.append('no_snap') if self.predictable: detected_options.append('predictable') if self.tsan: detected_options.append('tsan') if self.ubsan_vptr: detected_options.append('ubsan_vptr') return '\n'.join(detected_options) class BaseTestRunner(object): def __init__(self, basedir=None): self.basedir = basedir or BASE_DIR self.outdir = None self.build_config = None self.mode_name = None self.mode_options = None def execute(self, sys_args=None): if sys_args is None: # pragma: no cover sys_args = sys.argv[1:] try: parser = self._create_parser() options, args = self._parse_args(parser, sys_args) self._load_build_config(options) try: self._process_default_options(options) self._process_options(options) except TestRunnerError: parser.print_help() raise args = self._parse_test_args(args) suites = self._get_suites(args, options.verbose) self._setup_env() return self._do_execute(suites, args, options) except TestRunnerError: return 1 except KeyboardInterrupt: return 2 def _create_parser(self): parser = optparse.OptionParser() parser.usage = '%prog [options] [tests]' parser.description = """TESTS: %s""" % (TEST_MAP["default"]) self._add_parser_default_options(parser) self._add_parser_options(parser) return parser def _add_parser_default_options(self, parser): parser.add_option("--gn", help="Scan out.gn for the last built" " configuration", default=False, action="store_true") parser.add_option("--outdir", help="Base directory with compile output", default="out") parser.add_option("--buildbot", help="DEPRECATED!", default=False, action="store_true") parser.add_option("--arch", help="The architecture to run tests for") parser.add_option("-m", "--mode", help="The test mode in which to run (uppercase for ninja" " and buildbot builds): %s" % MODES.keys()) parser.add_option("--shell-dir", help="DEPRECATED! Executables from build " "directory will be used") parser.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") parser.add_option("--shard-count", help="Split tests into this number of shards", default=1, type="int") parser.add_option("--shard-run", help="Run this shard from the split up tests.", default=1, type="int") def _add_parser_options(self, parser): pass def _parse_args(self, parser, sys_args): options, args = parser.parse_args(sys_args) if any(map(lambda v: v and ',' in v, [options.arch, options.mode])): # pragma: no cover print 'Multiple arch/mode are deprecated' raise TestRunnerError() return options, args def _load_build_config(self, options): for outdir in self._possible_outdirs(options): try: self.build_config = self._do_load_build_config(outdir, options.verbose) except TestRunnerError: pass if not self.build_config: # pragma: no cover print 'Failed to load build config' raise TestRunnerError print 'Build found: %s' % self.outdir if str(self.build_config): print '>>> Autodetected:' print self.build_config # Returns possible build paths in order: # gn # outdir # outdir/arch.mode # Each path is provided in two versions: and /mode for buildbot. def _possible_outdirs(self, options): def outdirs(): if options.gn: yield self._get_gn_outdir() return yield options.outdir if options.arch and options.mode: yield os.path.join(options.outdir, '%s.%s' % (options.arch, options.mode)) for outdir in outdirs(): yield os.path.join(self.basedir, outdir) # buildbot option if options.mode: yield os.path.join(self.basedir, outdir, options.mode) def _get_gn_outdir(self): gn_out_dir = os.path.join(self.basedir, DEFAULT_OUT_GN) latest_timestamp = -1 latest_config = None for gn_config in os.listdir(gn_out_dir): gn_config_dir = os.path.join(gn_out_dir, gn_config) if not os.path.isdir(gn_config_dir): continue if os.path.getmtime(gn_config_dir) > latest_timestamp: latest_timestamp = os.path.getmtime(gn_config_dir) latest_config = gn_config if latest_config: print(">>> Latest GN build found: %s" % latest_config) return os.path.join(DEFAULT_OUT_GN, latest_config) def _do_load_build_config(self, outdir, verbose=False): build_config_path = os.path.join(outdir, "v8_build_config.json") if not os.path.exists(build_config_path): if verbose: print("Didn't find build config: %s" % build_config_path) raise TestRunnerError() with open(build_config_path) as f: try: build_config_json = json.load(f) except Exception: # pragma: no cover print("%s exists but contains invalid json. Is your build up-to-date?" % build_config_path) raise TestRunnerError() # In auto-detect mode the outdir is always where we found the build config. # This ensures that we'll also take the build products from there. self.outdir = os.path.dirname(build_config_path) return BuildConfig(build_config_json) def _process_default_options(self, options): # We don't use the mode for more path-magic. # Therefore transform the buildbot mode here to fix build_config value. if options.mode: options.mode = self._buildbot_to_v8_mode(options.mode) build_config_mode = 'debug' if self.build_config.is_debug else 'release' if options.mode: if options.mode not in MODES: # pragma: no cover print '%s mode is invalid' % options.mode raise TestRunnerError() if MODES[options.mode].execution_mode != build_config_mode: print ('execution mode (%s) for %s is inconsistent with build config ' '(%s)' % ( MODES[options.mode].execution_mode, options.mode, build_config_mode)) raise TestRunnerError() self.mode_name = options.mode else: self.mode_name = build_config_mode self.mode_options = MODES[self.mode_name] if options.arch and options.arch != self.build_config.arch: print('--arch value (%s) inconsistent with build config (%s).' % ( options.arch, self.build_config.arch)) raise TestRunnerError() if options.shell_dir: # pragma: no cover print('Warning: --shell-dir is deprecated. Searching for executables in ' 'build directory (%s) instead.' % self.outdir) def _buildbot_to_v8_mode(self, config): """Convert buildbot build configs to configs understood by the v8 runner. V8 configs are always lower case and without the additional _x64 suffix for 64 bit builds on windows with ninja. """ mode = config[:-4] if config.endswith('_x64') else config return mode.lower() def _process_options(self, options): pass def _setup_env(self): # Use the v8 root as cwd as some test cases use "load" with relative paths. os.chdir(self.basedir) # Many tests assume an English interface. os.environ['LANG'] = 'en_US.UTF-8' symbolizer_option = self._get_external_symbolizer_option() if self.build_config.asan: asan_options = [ symbolizer_option, 'allow_user_segv_handler=1', 'allocator_may_return_null=1', ] if not utils.GuessOS() in ['macos', 'windows']: # LSAN is not available on mac and windows. asan_options.append('detect_leaks=1') else: asan_options.append('detect_leaks=0') os.environ['ASAN_OPTIONS'] = ":".join(asan_options) if self.build_config.cfi_vptr: os.environ['UBSAN_OPTIONS'] = ":".join([ 'print_stacktrace=1', 'print_summary=1', 'symbolize=1', symbolizer_option, ]) if self.build_config.ubsan_vptr: os.environ['UBSAN_OPTIONS'] = ":".join([ 'print_stacktrace=1', symbolizer_option, ]) if self.build_config.msan: os.environ['MSAN_OPTIONS'] = symbolizer_option if self.build_config.tsan: suppressions_file = os.path.join( self.basedir, 'tools', 'sanitizers', 'tsan_suppressions.txt') os.environ['TSAN_OPTIONS'] = " ".join([ symbolizer_option, 'suppressions=%s' % suppressions_file, 'exit_code=0', 'report_thread_leaks=0', 'history_size=7', 'report_destroy_locked=0', ]) def _get_external_symbolizer_option(self): external_symbolizer_path = os.path.join( self.basedir, 'third_party', 'llvm-build', 'Release+Asserts', 'bin', 'llvm-symbolizer', ) if utils.IsWindows(): # Quote, because sanitizers might confuse colon as option separator. external_symbolizer_path = '"%s.exe"' % external_symbolizer_path return 'external_symbolizer_path=%s' % external_symbolizer_path def _parse_test_args(self, args): if not args: args = self._get_default_suite_names() # Expand arguments with grouped tests. The args should reflect the list # of suites as otherwise filters would break. def expand_test_group(name): return TEST_MAP.get(name, [name]) return reduce(list.__add__, map(expand_test_group, args), []) def _get_suites(self, args, verbose=False): names = self._args_to_suite_names(args) return self._load_suites(names, verbose) def _args_to_suite_names(self, args): # Use default tests if no test configuration was provided at the cmd line. all_names = set(utils.GetSuitePaths(os.path.join(self.basedir, 'test'))) args_names = OrderedDict([(arg.split('/')[0], None) for arg in args]) # set return [name for name in args_names if name in all_names] def _get_default_suite_names(self): return [] def _expand_test_group(self, name): return TEST_MAP.get(name, [name]) def _load_suites(self, names, verbose=False): def load_suite(name): if verbose: print '>>> Loading test suite: %s' % name return testsuite.TestSuite.LoadTestSuite( os.path.join(self.basedir, 'test', name)) return map(load_suite, names) # TODO(majeski): remove options & args parameters def _do_execute(self, suites, args, options): raise NotImplementedError() def _create_shard_proc(self, options): myid, count = self._get_shard_info(options) if count == 1: return None return ShardProc(myid - 1, count) def _get_shard_info(self, options): """ Returns pair: (id of the current shard [1; number of shards], number of shards) """ # Read gtest shard configuration from environment (e.g. set by swarming). # If none is present, use values passed on the command line. shard_count = int( os.environ.get('GTEST_TOTAL_SHARDS', options.shard_count)) shard_run = os.environ.get('GTEST_SHARD_INDEX') if shard_run is not None: # The v8 shard_run starts at 1, while GTEST_SHARD_INDEX starts at 0. shard_run = int(shard_run) + 1 else: shard_run = options.shard_run if options.shard_count > 1: # Log if a value was passed on the cmd line and it differs from the # environment variables. if options.shard_count != shard_count: # pragma: no cover print("shard_count from cmd line differs from environment variable " "GTEST_TOTAL_SHARDS") if (options.shard_run > 1 and options.shard_run != shard_run): # pragma: no cover print("shard_run from cmd line differs from environment variable " "GTEST_SHARD_INDEX") if shard_run < 1 or shard_run > shard_count: # TODO(machenbach): Turn this into an assert. If that's wrong on the # bots, printing will be quite useless. Or refactor this code to make # sure we get a return code != 0 after testing if we got here. print "shard-run not a valid number, should be in [1:shard-count]" print "defaulting back to running all tests" return 1, 1 return shard_run, shard_count