diff options
Diffstat (limited to 'deps/v8/tools/testrunner')
| -rw-r--r-- | deps/v8/tools/testrunner/base_runner.py | 15 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/command.py | 19 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/junit_output.py | 49 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/testsuite.py | 7 | ||||
| -rwxr-xr-x | deps/v8/tools/testrunner/local/testsuite_unittest.py | 3 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/variants.py | 11 | ||||
| -rwxr-xr-x | deps/v8/tools/testrunner/num_fuzzer.py | 4 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/outproc/mkgrokdump.py | 2 | ||||
| -rwxr-xr-x | deps/v8/tools/testrunner/standard_runner.py | 618 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/testproc/progress.py | 45 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/testproc/timeout.py | 1 |
11 files changed, 361 insertions, 413 deletions
diff --git a/deps/v8/tools/testrunner/base_runner.py b/deps/v8/tools/testrunner/base_runner.py index e31eb69ca5..214268631a 100644 --- a/deps/v8/tools/testrunner/base_runner.py +++ b/deps/v8/tools/testrunner/base_runner.py @@ -244,6 +244,11 @@ class BaseTestRunner(object): self.mode_options = None self.target_os = None + @property + def framework_name(self): + """String name of the base-runner subclass, used in test results.""" + raise NotImplementedError() + def execute(self, sys_args=None): if sys_args is None: # pragma: no cover sys_args = sys.argv[1:] @@ -336,9 +341,6 @@ class BaseTestRunner(object): "color, mono)") parser.add_option("--json-test-results", help="Path to a file for storing json results.") - parser.add_option("--junitout", help="File name of the JUnit output") - parser.add_option("--junittestsuite", default="v8tests", - help="The testsuite name in the JUnit output file") parser.add_option("--exit-after-n-failures", type="int", default=100, help="Exit after the first N failures instead of " "running all tests. Pass 0 to disable this feature.") @@ -630,7 +632,8 @@ class BaseTestRunner(object): if options.verbose: print('>>> Loading test suite: %s' % name) suite = testsuite.TestSuite.Load( - os.path.join(options.test_root, name), test_config) + os.path.join(options.test_root, name), test_config, + self.framework_name) if self._is_testsuite_supported(suite, options): tests = suite.load_tests_from_disk(variables) @@ -773,11 +776,9 @@ class BaseTestRunner(object): def _create_progress_indicators(self, test_count, options): procs = [PROGRESS_INDICATORS[options.progress]()] - if options.junitout: - procs.append(progress.JUnitTestProgressIndicator(options.junitout, - options.junittestsuite)) if options.json_test_results: procs.append(progress.JsonTestProgressIndicator( + self.framework_name, options.json_test_results, self.build_config.arch, self.mode_options.execution_mode)) diff --git a/deps/v8/tools/testrunner/local/command.py b/deps/v8/tools/testrunner/local/command.py index 8c0264e335..5eb0d8b20a 100644 --- a/deps/v8/tools/testrunner/local/command.py +++ b/deps/v8/tools/testrunner/local/command.py @@ -155,6 +155,25 @@ class BaseCommand(object): class PosixCommand(BaseCommand): + # TODO(machenbach): Use base process start without shell once + # https://crbug.com/v8/8889 is resolved. + def _start_process(self): + def wrapped(arg): + if set('() \'"') & set(arg): + return "'%s'" % arg.replace("'", "'\"'\"'") + return arg + try: + return subprocess.Popen( + args=' '.join(map(wrapped, self._get_popen_args())), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._get_env(), + shell=True, + ) + except Exception as e: + sys.stderr.write('Error executing: %s\n' % self) + raise e + def _kill_process(self, process): process.kill() diff --git a/deps/v8/tools/testrunner/local/junit_output.py b/deps/v8/tools/testrunner/local/junit_output.py deleted file mode 100644 index 52f31ec422..0000000000 --- a/deps/v8/tools/testrunner/local/junit_output.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2013 the V8 project authors. All rights reserved. -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials provided -# with the distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import xml.etree.ElementTree as xml - - -class JUnitTestOutput: - def __init__(self, test_suite_name): - self.root = xml.Element("testsuite") - self.root.attrib["name"] = test_suite_name - - def HasRunTest(self, test_name, test_cmd, test_duration, test_failure): - testCaseElement = xml.Element("testcase") - testCaseElement.attrib["name"] = test_name - testCaseElement.attrib["cmd"] = test_cmd - testCaseElement.attrib["time"] = str(round(test_duration, 3)) - if len(test_failure): - failureElement = xml.Element("failure") - failureElement.text = test_failure - testCaseElement.append(failureElement) - self.root.append(testCaseElement) - - def FinishAndWrite(self, f): - xml.ElementTree(self.root).write(f, "UTF-8") diff --git a/deps/v8/tools/testrunner/local/testsuite.py b/deps/v8/tools/testrunner/local/testsuite.py index 8656c1636d..1bfc0317fe 100644 --- a/deps/v8/tools/testrunner/local/testsuite.py +++ b/deps/v8/tools/testrunner/local/testsuite.py @@ -241,15 +241,16 @@ def _load_testsuite_module(name, root): class TestSuite(object): @staticmethod - def Load(root, test_config): + def Load(root, test_config, framework_name): name = root.split(os.path.sep)[-1] with _load_testsuite_module(name, root) as module: - return module.GetSuite(name, root, test_config) + return module.GetSuite(name, root, test_config, framework_name) - def __init__(self, name, root, test_config): + def __init__(self, name, root, test_config, framework_name): self.name = name # string self.root = root # string containing path self.test_config = test_config + self.framework_name = framework_name # name of the test runner impl self.tests = None # list of TestCase objects self.statusfile = None diff --git a/deps/v8/tools/testrunner/local/testsuite_unittest.py b/deps/v8/tools/testrunner/local/testsuite_unittest.py index 1cca79c205..b74fef1842 100755 --- a/deps/v8/tools/testrunner/local/testsuite_unittest.py +++ b/deps/v8/tools/testrunner/local/testsuite_unittest.py @@ -37,7 +37,8 @@ class TestSuiteTest(unittest.TestCase): verbose=False, ) - self.suite = TestSuite.Load(self.test_root, self.test_config) + self.suite = TestSuite.Load( + self.test_root, self.test_config, "standard_runner") def testLoadingTestSuites(self): self.assertEquals(self.suite.name, "fake_testsuite") diff --git a/deps/v8/tools/testrunner/local/variants.py b/deps/v8/tools/testrunner/local/variants.py index b3e446fb3f..ed9b1b87f5 100644 --- a/deps/v8/tools/testrunner/local/variants.py +++ b/deps/v8/tools/testrunner/local/variants.py @@ -7,19 +7,22 @@ ALL_VARIANT_FLAGS = { "code_serializer": [["--cache=code"]], "default": [[]], "future": [["--future"]], - "gc_stats": [["--gc_stats=1"]], + "gc_stats": [["--gc-stats=1"]], # Alias of exhaustive variants, but triggering new test framework features. "infra_staging": [[]], "interpreted_regexp": [["--regexp-interpret-all"]], "jitless": [["--jitless"]], - "no_liftoff": [["--no-wasm-tier-up"]], "minor_mc": [["--minor-mc"]], # No optimization means disable all optimizations. OptimizeFunctionOnNextCall # would not force optimization too. It turns into a Nop. Please see # https://chromium-review.googlesource.com/c/452620/ for more discussion. - "nooptimization": [["--noopt"]], + # For WebAssembly, we test "Liftoff-only" in the nooptimization variant and + # "TurboFan-only" in the stress variant. The WebAssembly configuration is + # independent of JS optimizations, so we can combine those configs. + "nooptimization": [["--no-opt", "--liftoff", "--no-wasm-tier-up"]], "slow_path": [["--force-slow-path"]], - "stress": [["--stress-opt", "--always-opt"]], + "stress": [["--stress-opt", "--always-opt", "--no-liftoff", + "--no-wasm-tier-up"]], "stress_background_compile": [["--stress-background-compile"]], "stress_incremental_marking": [["--stress-incremental-marking"]], # Trigger stress sampling allocation profiler with sample interval = 2^14 diff --git a/deps/v8/tools/testrunner/num_fuzzer.py b/deps/v8/tools/testrunner/num_fuzzer.py index e51966b5cf..d4e92a61e8 100755 --- a/deps/v8/tools/testrunner/num_fuzzer.py +++ b/deps/v8/tools/testrunner/num_fuzzer.py @@ -33,6 +33,10 @@ class NumFuzzer(base_runner.BaseTestRunner): def __init__(self, *args, **kwargs): super(NumFuzzer, self).__init__(*args, **kwargs) + @property + def framework_name(self): + return 'num_fuzzer' + def _add_parser_options(self, parser): parser.add_option("--fuzzer-random-seed", default=0, help="Default seed for initializing fuzzer random " diff --git a/deps/v8/tools/testrunner/outproc/mkgrokdump.py b/deps/v8/tools/testrunner/outproc/mkgrokdump.py index 8efde1226f..4013023776 100644 --- a/deps/v8/tools/testrunner/outproc/mkgrokdump.py +++ b/deps/v8/tools/testrunner/outproc/mkgrokdump.py @@ -20,7 +20,7 @@ class OutProc(base.OutProc): diff = difflib.unified_diff(expected_lines, actual_lines, lineterm="", fromfile="expected_path") diffstring = '\n'.join(diff) - if diffstring is not "": + if diffstring != "": if "generated from a non-shipping build" in output.stdout: return False if not "generated from a shipping build" in output.stdout: diff --git a/deps/v8/tools/testrunner/standard_runner.py b/deps/v8/tools/testrunner/standard_runner.py index 300340ed98..2a08d2d97e 100755 --- a/deps/v8/tools/testrunner/standard_runner.py +++ b/deps/v8/tools/testrunner/standard_runner.py @@ -29,34 +29,34 @@ from testrunner.utils import random_utils ARCH_GUESS = utils.DefaultArch() -VARIANTS = ["default"] +VARIANTS = ['default'] MORE_VARIANTS = [ - "jitless", - "stress", - "stress_background_compile", - "stress_incremental_marking", + 'jitless', + 'stress', + 'stress_background_compile', + 'stress_incremental_marking', ] VARIANT_ALIASES = { # The default for developer workstations. - "dev": VARIANTS, + 'dev': VARIANTS, # Additional variants, run on all bots. - "more": MORE_VARIANTS, - # Shortcut for the two above ("more" first - it has the longer running tests). - "exhaustive": MORE_VARIANTS + VARIANTS, + 'more': MORE_VARIANTS, + # Shortcut for the two above ('more' first - it has the longer running tests). + 'exhaustive': MORE_VARIANTS + VARIANTS, # Additional variants, run on a subset of bots. - "extra": ["nooptimization", "future", "no_liftoff", "no_wasm_traps"], + 'extra': ['nooptimization', 'future', 'no_wasm_traps'], } -GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", - "--concurrent-recompilation-queue-length=64", - "--concurrent-recompilation-delay=500", - "--concurrent-recompilation", - "--stress-flush-bytecode"] +GC_STRESS_FLAGS = ['--gc-interval=500', '--stress-compaction', + '--concurrent-recompilation-queue-length=64', + '--concurrent-recompilation-delay=500', + '--concurrent-recompilation', + '--stress-flush-bytecode'] -RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000", - "--stress-compaction-random"] +RANDOM_GC_STRESS_FLAGS = ['--random-gc-interval=5000', + '--stress-compaction-random'] PREDICTABLE_WRAPPER = os.path.join( @@ -64,296 +64,300 @@ PREDICTABLE_WRAPPER = os.path.join( class StandardTestRunner(base_runner.BaseTestRunner): - def __init__(self, *args, **kwargs): - super(StandardTestRunner, self).__init__(*args, **kwargs) - - self.sancov_dir = None - self._variants = None - - def _get_default_suite_names(self): - return ['default'] - - def _add_parser_options(self, parser): - parser.add_option("--novfp3", - help="Indicates that V8 was compiled without VFP3" - " support", - default=False, action="store_true") - - # Variants - parser.add_option("--no-variants", "--novariants", - help="Deprecated. " - "Equivalent to passing --variants=default", - default=False, dest="no_variants", action="store_true") - parser.add_option("--variants", - help="Comma-separated list of testing variants;" - " default: \"%s\"" % ",".join(VARIANTS)) - parser.add_option("--exhaustive-variants", - default=False, action="store_true", - help="Deprecated. " - "Equivalent to passing --variants=exhaustive") - - # Filters - parser.add_option("--slow-tests", default="dontcare", - help="Regard slow tests (run|skip|dontcare)") - parser.add_option("--pass-fail-tests", default="dontcare", - help="Regard pass|fail tests (run|skip|dontcare)") - parser.add_option("--quickcheck", default=False, action="store_true", - help=("Quick check mode (skip slow tests)")) - parser.add_option("--dont-skip-slow-simulator-tests", - help="Don't skip more slow tests when using a" - " simulator.", - default=False, action="store_true", - dest="dont_skip_simulator_slow_tests") - - # Stress modes - parser.add_option("--gc-stress", - help="Switch on GC stress mode", - default=False, action="store_true") - parser.add_option("--random-gc-stress", - help="Switch on random GC stress mode", - default=False, action="store_true") - parser.add_option("--random-seed-stress-count", default=1, type="int", - dest="random_seed_stress_count", - help="Number of runs with different random seeds. Only " - "with test processors: 0 means infinite " - "generation.") - - # Noop - parser.add_option("--cfi-vptr", - help="Run tests with UBSAN cfi_vptr option.", - default=False, action="store_true") - parser.add_option("--infra-staging", help="Use new test runner features", - dest='infra_staging', default=None, - action="store_true") - parser.add_option("--no-infra-staging", - help="Opt out of new test runner features", - dest='infra_staging', default=None, - action="store_false") - parser.add_option("--no-sorting", "--nosorting", - help="Don't sort tests according to duration of last" - " run.", - default=False, dest="no_sorting", action="store_true") - parser.add_option("--no-presubmit", "--nopresubmit", - help='Skip presubmit checks (deprecated)', - default=False, dest="no_presubmit", action="store_true") - - # Unimplemented for test processors - parser.add_option("--sancov-dir", - help="Directory where to collect coverage data") - parser.add_option("--cat", help="Print the source of the tests", - default=False, action="store_true") - parser.add_option("--flakiness-results", - help="Path to a file for storing flakiness json.") - parser.add_option("--time", help="Print timing information after running", - default=False, action="store_true") - parser.add_option("--warn-unused", help="Report unused rules", - default=False, action="store_true") - parser.add_option("--report", default=False, action="store_true", - help="Print a summary of the tests to be run") - - - def _process_options(self, options): - if options.sancov_dir: - self.sancov_dir = options.sancov_dir - if not os.path.exists(self.sancov_dir): - print("sancov-dir %s doesn't exist" % self.sancov_dir) - raise base_runner.TestRunnerError() - - if options.gc_stress: - options.extra_flags += GC_STRESS_FLAGS - - if options.random_gc_stress: - options.extra_flags += RANDOM_GC_STRESS_FLAGS - - if self.build_config.asan: - options.extra_flags.append("--invoke-weak-callbacks") - options.extra_flags.append("--omit-quit") - - if self.build_config.no_snap: - # Speed up slow nosnap runs. Allocation verification is covered by - # running mksnapshot on other builders. - options.extra_flags.append("--no-turbo-verify-allocation") - - if options.novfp3: - options.extra_flags.append("--noenable-vfp3") - - if options.no_variants: # pragma: no cover - print ("Option --no-variants is deprecated. " - "Pass --variants=default instead.") - assert not options.variants - options.variants = "default" - - if options.exhaustive_variants: # pragma: no cover - # TODO(machenbach): Switch infra to --variants=exhaustive after M65. - print ("Option --exhaustive-variants is deprecated. " - "Pass --variants=exhaustive instead.") - # This is used on many bots. It includes a larger set of default - # variants. - # Other options for manipulating variants still apply afterwards. - assert not options.variants - options.variants = "exhaustive" - - if options.quickcheck: - assert not options.variants - options.variants = "stress,default" - options.slow_tests = "skip" - options.pass_fail_tests = "skip" - - if self.build_config.predictable: - options.variants = "default" - options.extra_flags.append("--predictable") - options.extra_flags.append("--verify_predictable") - options.extra_flags.append("--no-inline-new") - # Add predictable wrapper to command prefix. - options.command_prefix = ( - [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) - - # TODO(machenbach): Figure out how to test a bigger subset of variants on - # msan. - if self.build_config.msan: - options.variants = "default" - - if options.variants == "infra_staging": - options.variants = "exhaustive" - - self._variants = self._parse_variants(options.variants) - - def CheckTestMode(name, option): # pragma: no cover - if not option in ["run", "skip", "dontcare"]: - print("Unknown %s mode %s" % (name, option)) - raise base_runner.TestRunnerError() - CheckTestMode("slow test", options.slow_tests) - CheckTestMode("pass|fail test", options.pass_fail_tests) - if self.build_config.no_i18n: - base_runner.TEST_MAP["bot_default"].remove("intl") - base_runner.TEST_MAP["default"].remove("intl") - # TODO(machenbach): uncomment after infra side lands. - # base_runner.TEST_MAP["d8_default"].remove("intl") - - def _parse_variants(self, aliases_str): - # Use developer defaults if no variant was specified. - aliases_str = aliases_str or 'dev' - aliases = aliases_str.split(',') - user_variants = set(reduce( - list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases])) - - result = [v for v in ALL_VARIANTS if v in user_variants] - if len(result) == len(user_variants): - return result - - for v in user_variants: - if v not in ALL_VARIANTS: - print('Unknown variant: %s' % v) - raise base_runner.TestRunnerError() - assert False, 'Unreachable' - - def _setup_env(self): - super(StandardTestRunner, self)._setup_env() - - symbolizer_option = self._get_external_symbolizer_option() - - if self.sancov_dir: - os.environ['ASAN_OPTIONS'] = ":".join([ - 'coverage=1', - 'coverage_dir=%s' % self.sancov_dir, - symbolizer_option, - "allow_user_segv_handler=1", - ]) - - def _get_statusfile_variables(self, options): - variables = ( - super(StandardTestRunner, self)._get_statusfile_variables(options)) - - simulator_run = ( - not options.dont_skip_simulator_slow_tests and - self.build_config.arch in [ - 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', - 'ppc64', 's390', 's390x'] and - bool(ARCH_GUESS) and - self.build_config.arch != ARCH_GUESS) - - variables.update({ - 'gc_stress': options.gc_stress or options.random_gc_stress, - 'gc_fuzzer': options.random_gc_stress, - 'novfp3': options.novfp3, - 'simulator_run': simulator_run, - }) - return variables - - def _do_execute(self, tests, args, options): - jobs = options.j - - print('>>> Running with test processors') - loader = LoadProc(tests) - results = self._create_result_tracker(options) - indicators = self._create_progress_indicators( - tests.test_count_estimate, options) - - outproc_factory = None - if self.build_config.predictable: - outproc_factory = predictable.get_outproc - execproc = ExecutionProc(jobs, outproc_factory) - sigproc = self._create_signal_proc() - - procs = [ - loader, - NameFilterProc(args) if args else None, - StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), - VariantProc(self._variants), - StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), - self._create_predictable_filter(), - self._create_shard_proc(options), - self._create_seed_proc(options), - sigproc, - ] + indicators + [ - results, - self._create_timeout_proc(options), - self._create_rerun_proc(options), - execproc, - ] - - self._prepare_procs(procs) - - loader.load_initial_tests(initial_batch_size=options.j*2) - - # This starts up worker processes and blocks until all tests are - # processed. - execproc.run() - - for indicator in indicators: - indicator.finished() - - - if tests.test_count_estimate: - percentage = float(results.total) / tests.test_count_estimate * 100 - else: - percentage = 0 - - print (('>>> %d base tests produced %d (%d%s)' - ' non-filtered tests') % ( - tests.test_count_estimate, results.total, percentage, '%')) - - print('>>> %d tests ran' % (results.total - results.remaining)) - - exit_code = utils.EXIT_CODE_PASS - if results.failed: - exit_code = utils.EXIT_CODE_FAILURES - if not results.total: - exit_code = utils.EXIT_CODE_NO_TESTS - - # Indicate if a SIGINT or SIGTERM happened. - return max(exit_code, sigproc.exit_code) - - def _create_predictable_filter(self): - if not self.build_config.predictable: - return None - return predictable.PredictableFilterProc() - - def _create_seed_proc(self, options): - if options.random_seed_stress_count == 1: - return None - return SeedProc(options.random_seed_stress_count, options.random_seed, - options.j * 4) + def __init__(self, *args, **kwargs): + super(StandardTestRunner, self).__init__(*args, **kwargs) + + self.sancov_dir = None + self._variants = None + + @property + def framework_name(self): + return 'standard_runner' + + def _get_default_suite_names(self): + return ['default'] + + def _add_parser_options(self, parser): + parser.add_option('--novfp3', + help='Indicates that V8 was compiled without VFP3' + ' support', + default=False, action='store_true') + + # Variants + parser.add_option('--no-variants', '--novariants', + help='Deprecated. ' + 'Equivalent to passing --variants=default', + default=False, dest='no_variants', action='store_true') + parser.add_option('--variants', + help='Comma-separated list of testing variants;' + ' default: "%s"' % ','.join(VARIANTS)) + parser.add_option('--exhaustive-variants', + default=False, action='store_true', + help='Deprecated. ' + 'Equivalent to passing --variants=exhaustive') + + # Filters + parser.add_option('--slow-tests', default='dontcare', + help='Regard slow tests (run|skip|dontcare)') + parser.add_option('--pass-fail-tests', default='dontcare', + help='Regard pass|fail tests (run|skip|dontcare)') + parser.add_option('--quickcheck', default=False, action='store_true', + help=('Quick check mode (skip slow tests)')) + parser.add_option('--dont-skip-slow-simulator-tests', + help='Don\'t skip more slow tests when using a' + ' simulator.', + default=False, action='store_true', + dest='dont_skip_simulator_slow_tests') + + # Stress modes + parser.add_option('--gc-stress', + help='Switch on GC stress mode', + default=False, action='store_true') + parser.add_option('--random-gc-stress', + help='Switch on random GC stress mode', + default=False, action='store_true') + parser.add_option('--random-seed-stress-count', default=1, type='int', + dest='random_seed_stress_count', + help='Number of runs with different random seeds. Only ' + 'with test processors: 0 means infinite ' + 'generation.') + + # Noop + parser.add_option('--cfi-vptr', + help='Run tests with UBSAN cfi_vptr option.', + default=False, action='store_true') + parser.add_option('--infra-staging', help='Use new test runner features', + dest='infra_staging', default=None, + action='store_true') + parser.add_option('--no-infra-staging', + help='Opt out of new test runner features', + dest='infra_staging', default=None, + action='store_false') + parser.add_option('--no-sorting', '--nosorting', + help='Don\'t sort tests according to duration of last' + ' run.', + default=False, dest='no_sorting', action='store_true') + parser.add_option('--no-presubmit', '--nopresubmit', + help='Skip presubmit checks (deprecated)', + default=False, dest='no_presubmit', action='store_true') + + # Unimplemented for test processors + parser.add_option('--sancov-dir', + help='Directory where to collect coverage data') + parser.add_option('--cat', help='Print the source of the tests', + default=False, action='store_true') + parser.add_option('--flakiness-results', + help='Path to a file for storing flakiness json.') + parser.add_option('--time', help='Print timing information after running', + default=False, action='store_true') + parser.add_option('--warn-unused', help='Report unused rules', + default=False, action='store_true') + parser.add_option('--report', default=False, action='store_true', + help='Print a summary of the tests to be run') + + + def _process_options(self, options): + if options.sancov_dir: + self.sancov_dir = options.sancov_dir + if not os.path.exists(self.sancov_dir): + print('sancov-dir %s doesn\'t exist' % self.sancov_dir) + raise base_runner.TestRunnerError() + + if options.gc_stress: + options.extra_flags += GC_STRESS_FLAGS + + if options.random_gc_stress: + options.extra_flags += RANDOM_GC_STRESS_FLAGS + + if self.build_config.asan: + options.extra_flags.append('--invoke-weak-callbacks') + options.extra_flags.append('--omit-quit') + + if self.build_config.no_snap: + # Speed up slow nosnap runs. Allocation verification is covered by + # running mksnapshot on other builders. + options.extra_flags.append('--no-turbo-verify-allocation') + + if options.novfp3: + options.extra_flags.append('--noenable-vfp3') + + if options.no_variants: # pragma: no cover + print ('Option --no-variants is deprecated. ' + 'Pass --variants=default instead.') + assert not options.variants + options.variants = 'default' + + if options.exhaustive_variants: # pragma: no cover + # TODO(machenbach): Switch infra to --variants=exhaustive after M65. + print ('Option --exhaustive-variants is deprecated. ' + 'Pass --variants=exhaustive instead.') + # This is used on many bots. It includes a larger set of default + # variants. + # Other options for manipulating variants still apply afterwards. + assert not options.variants + options.variants = 'exhaustive' + + if options.quickcheck: + assert not options.variants + options.variants = 'stress,default' + options.slow_tests = 'skip' + options.pass_fail_tests = 'skip' + + if self.build_config.predictable: + options.variants = 'default' + options.extra_flags.append('--predictable') + options.extra_flags.append('--verify-predictable') + options.extra_flags.append('--no-inline-new') + # Add predictable wrapper to command prefix. + options.command_prefix = ( + [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix) + + # TODO(machenbach): Figure out how to test a bigger subset of variants on + # msan. + if self.build_config.msan: + options.variants = 'default' + + if options.variants == 'infra_staging': + options.variants = 'exhaustive' + + self._variants = self._parse_variants(options.variants) + + def CheckTestMode(name, option): # pragma: no cover + if not option in ['run', 'skip', 'dontcare']: + print('Unknown %s mode %s' % (name, option)) + raise base_runner.TestRunnerError() + CheckTestMode('slow test', options.slow_tests) + CheckTestMode('pass|fail test', options.pass_fail_tests) + if self.build_config.no_i18n: + base_runner.TEST_MAP['bot_default'].remove('intl') + base_runner.TEST_MAP['default'].remove('intl') + # TODO(machenbach): uncomment after infra side lands. + # base_runner.TEST_MAP['d8_default'].remove('intl') + + def _parse_variants(self, aliases_str): + # Use developer defaults if no variant was specified. + aliases_str = aliases_str or 'dev' + aliases = aliases_str.split(',') + user_variants = set(reduce( + list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases])) + + result = [v for v in ALL_VARIANTS if v in user_variants] + if len(result) == len(user_variants): + return result + + for v in user_variants: + if v not in ALL_VARIANTS: + print('Unknown variant: %s' % v) + raise base_runner.TestRunnerError() + assert False, 'Unreachable' + + def _setup_env(self): + super(StandardTestRunner, self)._setup_env() + + symbolizer_option = self._get_external_symbolizer_option() + + if self.sancov_dir: + os.environ['ASAN_OPTIONS'] = ':'.join([ + 'coverage=1', + 'coverage_dir=%s' % self.sancov_dir, + symbolizer_option, + 'allow_user_segv_handler=1', + ]) + + def _get_statusfile_variables(self, options): + variables = ( + super(StandardTestRunner, self)._get_statusfile_variables(options)) + + simulator_run = ( + not options.dont_skip_simulator_slow_tests and + self.build_config.arch in [ + 'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc', + 'ppc64', 's390', 's390x'] and + bool(ARCH_GUESS) and + self.build_config.arch != ARCH_GUESS) + + variables.update({ + 'gc_stress': options.gc_stress or options.random_gc_stress, + 'gc_fuzzer': options.random_gc_stress, + 'novfp3': options.novfp3, + 'simulator_run': simulator_run, + }) + return variables + + def _do_execute(self, tests, args, options): + jobs = options.j + + print('>>> Running with test processors') + loader = LoadProc(tests) + results = self._create_result_tracker(options) + indicators = self._create_progress_indicators( + tests.test_count_estimate, options) + + outproc_factory = None + if self.build_config.predictable: + outproc_factory = predictable.get_outproc + execproc = ExecutionProc(jobs, outproc_factory) + sigproc = self._create_signal_proc() + + procs = [ + loader, + NameFilterProc(args) if args else None, + StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), + VariantProc(self._variants), + StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), + self._create_predictable_filter(), + self._create_shard_proc(options), + self._create_seed_proc(options), + sigproc, + ] + indicators + [ + results, + self._create_timeout_proc(options), + self._create_rerun_proc(options), + execproc, + ] + + self._prepare_procs(procs) + + loader.load_initial_tests(initial_batch_size=options.j*2) + + # This starts up worker processes and blocks until all tests are + # processed. + execproc.run() + + for indicator in indicators: + indicator.finished() + + + if tests.test_count_estimate: + percentage = float(results.total) / tests.test_count_estimate * 100 + else: + percentage = 0 + + print (('>>> %d base tests produced %d (%d%s)' + ' non-filtered tests') % ( + tests.test_count_estimate, results.total, percentage, '%')) + + print('>>> %d tests ran' % (results.total - results.remaining)) + + exit_code = utils.EXIT_CODE_PASS + if results.failed: + exit_code = utils.EXIT_CODE_FAILURES + if not results.total: + exit_code = utils.EXIT_CODE_NO_TESTS + + # Indicate if a SIGINT or SIGTERM happened. + return max(exit_code, sigproc.exit_code) + + def _create_predictable_filter(self): + if not self.build_config.predictable: + return None + return predictable.PredictableFilterProc() + + def _create_seed_proc(self, options): + if options.random_seed_stress_count == 1: + return None + return SeedProc(options.random_seed_stress_count, options.random_seed, + options.j * 4) if __name__ == '__main__': diff --git a/deps/v8/tools/testrunner/testproc/progress.py b/deps/v8/tools/testrunner/testproc/progress.py index daa05c4a21..46e3a2e4d8 100644 --- a/deps/v8/tools/testrunner/testproc/progress.py +++ b/deps/v8/tools/testrunner/testproc/progress.py @@ -11,7 +11,6 @@ import sys import time from . import base -from ..local import junit_output def print_failure_header(test): @@ -259,47 +258,8 @@ class MonochromeProgressIndicator(CompactProgressIndicator): print(("\r" + (" " * last_length) + "\r"), end='') -class JUnitTestProgressIndicator(ProgressIndicator): - def __init__(self, junitout, junittestsuite): - super(JUnitTestProgressIndicator, self).__init__() - self._requirement = base.DROP_PASS_STDOUT - - self.outputter = junit_output.JUnitTestOutput(junittestsuite) - if junitout: - self.outfile = open(junitout, "w") - else: - self.outfile = sys.stdout - - def _on_result_for(self, test, result): - # TODO(majeski): Support for dummy/grouped results - fail_text = "" - output = result.output - if result.has_unexpected_output: - stdout = output.stdout.strip() - if len(stdout): - fail_text += "stdout:\n%s\n" % stdout - stderr = output.stderr.strip() - if len(stderr): - fail_text += "stderr:\n%s\n" % stderr - fail_text += "Command: %s" % result.cmd.to_string() - if output.HasCrashed(): - fail_text += "exit code: %d\n--- CRASHED ---" % output.exit_code - if output.HasTimedOut(): - fail_text += "--- TIMEOUT ---" - self.outputter.HasRunTest( - test_name=str(test), - test_cmd=result.cmd.to_string(relative=True), - test_duration=output.duration, - test_failure=fail_text) - - def finished(self): - self.outputter.FinishAndWrite(self.outfile) - if self.outfile != sys.stdout: - self.outfile.close() - - class JsonTestProgressIndicator(ProgressIndicator): - def __init__(self, json_test_results, arch, mode): + def __init__(self, framework_name, json_test_results, arch, mode): super(JsonTestProgressIndicator, self).__init__() # We want to drop stdout/err for all passed tests on the first try, but we # need to get outputs for all runs after the first one. To accommodate that, @@ -307,6 +267,7 @@ class JsonTestProgressIndicator(ProgressIndicator): # keep_output set to True in the RerunProc. self._requirement = base.DROP_PASS_STDOUT + self.framework_name = framework_name self.json_test_results = json_test_results self.arch = arch self.mode = mode @@ -347,6 +308,8 @@ class JsonTestProgressIndicator(ProgressIndicator): "random_seed": test.random_seed, "target_name": test.get_shell(), "variant": test.variant, + "variant_flags": test.variant_flags, + "framework_name": self.framework_name, }) def finished(self): diff --git a/deps/v8/tools/testrunner/testproc/timeout.py b/deps/v8/tools/testrunner/testproc/timeout.py index 84ddc656e2..54dc60e9b4 100644 --- a/deps/v8/tools/testrunner/testproc/timeout.py +++ b/deps/v8/tools/testrunner/testproc/timeout.py @@ -25,4 +25,5 @@ class TimeoutProc(base.TestProcObserver): def _on_event(self): if not self.is_stopped: if time.time() - self._start > self._duration_sec: + print('>>> Total timeout reached.') self.stop() |