diff options
Diffstat (limited to 'deps/v8/tools/testrunner')
| -rw-r--r-- | deps/v8/tools/testrunner/base_runner.py | 80 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/android.py | 21 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/fake_testsuite/fake_testsuite.status | 5 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/fake_testsuite/testcfg.py | 23 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/statusfile.py | 3 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/local/testsuite.py | 119 | ||||
| -rwxr-xr-x | deps/v8/tools/testrunner/local/testsuite_unittest.py | 128 | ||||
| -rwxr-xr-x | deps/v8/tools/testrunner/num_fuzzer.py | 36 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/objects/testcase.py | 9 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/outproc/test262.py | 35 | ||||
| -rwxr-xr-x | deps/v8/tools/testrunner/standard_runner.py | 11 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/test_config.py | 2 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/testproc/fuzzer.py | 11 | ||||
| -rw-r--r-- | deps/v8/tools/testrunner/testproc/progress.py | 25 |
14 files changed, 198 insertions, 310 deletions
diff --git a/deps/v8/tools/testrunner/base_runner.py b/deps/v8/tools/testrunner/base_runner.py index fcb2202f8a..8c5f7ad205 100644 --- a/deps/v8/tools/testrunner/base_runner.py +++ b/deps/v8/tools/testrunner/base_runner.py @@ -51,6 +51,7 @@ TEST_MAP = { "inspector", "webkit", "mkgrokdump", + "wasm-js", "fuzzer", "message", "preparser", @@ -65,6 +66,7 @@ TEST_MAP = { "wasm-spec-tests", "inspector", "mkgrokdump", + "wasm-js", "fuzzer", "message", "preparser", @@ -183,6 +185,10 @@ class BuildConfig(object): self.predictable = build_config['v8_enable_verify_predictable'] self.tsan = build_config['is_tsan'] self.ubsan_vptr = build_config['is_ubsan_vptr'] + self.embedded_builtins = build_config['v8_enable_embedded_builtins'] + self.verify_csa = build_config['v8_enable_verify_csa'] + self.lite_mode = build_config['v8_enable_lite_mode'] + self.pointer_compression = build_config['v8_enable_pointer_compression'] # Export only for MIPS target if self.arch in ['mips', 'mipsel', 'mips64', 'mips64el']: self.mips_arch_variant = build_config['mips_arch_variant'] @@ -211,6 +217,14 @@ class BuildConfig(object): detected_options.append('tsan') if self.ubsan_vptr: detected_options.append('ubsan_vptr') + if self.embedded_builtins: + detected_options.append('embedded_builtins') + if self.verify_csa: + detected_options.append('verify_csa') + if self.lite_mode: + detected_options.append('lite_mode') + if self.pointer_compression: + detected_options.append('pointer_compression') return '\n'.join(detected_options) @@ -246,14 +260,10 @@ class BaseTestRunner(object): raise args = self._parse_test_args(args) - suites = self._get_suites(args, options) - self._prepare_suites(suites, options) - + tests = self._load_testsuite_generators(args, options) self._setup_env() - print(">>> Running tests for %s.%s" % (self.build_config.arch, self.mode_name)) - tests = [t for s in suites for t in s.tests] return self._do_execute(tests, args, options) except TestRunnerError: return utils.EXIT_CODE_INTERNAL_ERROR @@ -334,6 +344,8 @@ class BaseTestRunner(object): help="Run without test harness of a given suite") parser.add_option("--random-seed", default=0, type=int, help="Default seed for initializing random generator") + parser.add_option("--run-skipped", help="Also run skipped tests.", + default=False, action="store_true") parser.add_option("-t", "--timeout", default=60, type=int, help="Timeout for single test in seconds") parser.add_option("-v", "--verbose", default=False, action="store_true", @@ -580,10 +592,6 @@ class BaseTestRunner(object): return reduce(list.__add__, map(expand_test_group, args), []) - def _get_suites(self, args, options): - names = self._args_to_suite_names(args, options.test_root) - return self._load_suites(names, options) - def _args_to_suite_names(self, args, test_root): # Use default tests if no test configuration was provided at the cmd line. all_names = set(utils.GetSuitePaths(test_root)) @@ -593,26 +601,34 @@ class BaseTestRunner(object): def _get_default_suite_names(self): return [] - def _load_suites(self, names, options): + def _load_testsuite_generators(self, args, options): + names = self._args_to_suite_names(args, options.test_root) test_config = self._create_test_config(options) - def load_suite(name): + variables = self._get_statusfile_variables(options) + slow_chain, fast_chain = [], [] + for name in names: if options.verbose: print '>>> Loading test suite: %s' % name - return testsuite.TestSuite.LoadTestSuite( - os.path.join(options.test_root, name), - test_config) - return map(load_suite, names) - - def _prepare_suites(self, suites, options): - self._load_status_files(suites, options) - for s in suites: - s.ReadTestCases() - - def _load_status_files(self, suites, options): - # simd_mips is true if SIMD is fully supported on MIPS - variables = self._get_statusfile_variables(options) - for s in suites: - s.ReadStatusFile(variables) + suite = testsuite.TestSuite.Load( + os.path.join(options.test_root, name), test_config) + + if self._is_testsuite_supported(suite, options): + slow_tests, fast_tests = suite.load_tests_from_disk(variables) + slow_chain.append(slow_tests) + fast_chain.append(fast_tests) + + for tests in slow_chain: + for test in tests: + yield test + + for tests in fast_chain: + for test in tests: + yield test + + def _is_testsuite_supported(self, suite, options): + """A predicate that can be overridden to filter out unsupported TestSuite + instances (see NumFuzzer for usage).""" + return True def _get_statusfile_variables(self, options): simd_mips = ( @@ -624,7 +640,6 @@ class BaseTestRunner(object): self.build_config.arch in ['mipsel', 'mips', 'mips64', 'mips64el'] and self.build_config.mips_arch_variant) - # TODO(all): Combine "simulator" and "simulator_run". # TODO(machenbach): In GN we can derive simulator run from # target_arch != v8_target_arch in the dumped build config. return { @@ -639,19 +654,25 @@ class BaseTestRunner(object): "gcov_coverage": self.build_config.gcov_coverage, "isolates": options.isolates, "mips_arch_variant": mips_arch_variant, - "mode": self.mode_options.status_mode, + "mode": self.mode_options.status_mode + if not self.build_config.dcheck_always_on + else "debug", "msan": self.build_config.msan, "no_harness": options.no_harness, "no_i18n": self.build_config.no_i18n, "no_snap": self.build_config.no_snap, "novfp3": False, + "optimize_for_size": "--optimize-for-size" in options.extra_flags, "predictable": self.build_config.predictable, "simd_mips": simd_mips, - "simulator": utils.UseSimulator(self.build_config.arch), "simulator_run": False, "system": self.target_os, "tsan": self.build_config.tsan, "ubsan_vptr": self.build_config.ubsan_vptr, + "embedded_builtins": self.build_config.embedded_builtins, + "verify_csa": self.build_config.verify_csa, + "lite_mode": self.build_config.lite_mode, + "pointer_compression": self.build_config.pointer_compression, } def _create_test_config(self, options): @@ -664,6 +685,7 @@ class BaseTestRunner(object): no_harness=options.no_harness, noi18n=self.build_config.no_i18n, random_seed=options.random_seed, + run_skipped=options.run_skipped, shell_dir=self.outdir, timeout=timeout, verbose=options.verbose, diff --git a/deps/v8/tools/testrunner/local/android.py b/deps/v8/tools/testrunner/local/android.py index 707614f095..5724f9ee2a 100644 --- a/deps/v8/tools/testrunner/local/android.py +++ b/deps/v8/tools/testrunner/local/android.py @@ -42,21 +42,16 @@ class _Driver(object): from devil.android import device_utils # pylint: disable=import-error from devil.android.perf import cache_control # pylint: disable=import-error from devil.android.perf import perf_control # pylint: disable=import-error - from devil.android.sdk import adb_wrapper # pylint: disable=import-error global cache_control global device_errors global perf_control devil_chromium.Initialize() - if not device: - # Detect attached device if not specified. - devices = adb_wrapper.AdbWrapper.Devices() - assert devices, 'No devices detected' - assert len(devices) == 1, 'Multiple devices detected.' - device = str(devices[0]) - self.adb_wrapper = adb_wrapper.AdbWrapper(device) - self.device = device_utils.DeviceUtils(self.adb_wrapper) + # Find specified device or a single attached device if none was specified. + # In case none or multiple devices are attached, this raises an exception. + self.device = device_utils.DeviceUtils.HealthyDevices( + retries=5, enable_usb_resets=True, device_arg=device)[0] # This remembers what we have already pushed to the device. self.pushed = set() @@ -77,6 +72,8 @@ class _Driver(object): skip_if_missing: Keeps silent about missing files when set. Otherwise logs error. """ + # TODO(sergiyb): Implement this method using self.device.PushChangedFiles to + # avoid accessing low-level self.device.adb. file_on_host = os.path.join(host_dir, file_name) # Only push files not yet pushed in one execution. @@ -95,13 +92,13 @@ class _Driver(object): # Work-around for 'text file busy' errors. Push the files to a temporary # location and then copy them with a shell command. - output = self.adb_wrapper.Push(file_on_host, file_on_device_tmp) + output = self.device.adb.Push(file_on_host, file_on_device_tmp) # Success looks like this: '3035 KB/s (12512056 bytes in 4.025s)'. # Errors look like this: 'failed to copy ... '. if output and not re.search('^[0-9]', output.splitlines()[-1]): logging.critical('PUSH FAILED: ' + output) - self.adb_wrapper.Shell('mkdir -p %s' % folder_on_device) - self.adb_wrapper.Shell('cp %s %s' % (file_on_device_tmp, file_on_device)) + self.device.adb.Shell('mkdir -p %s' % folder_on_device) + self.device.adb.Shell('cp %s %s' % (file_on_device_tmp, file_on_device)) self.pushed.add(file_on_host) def push_executable(self, shell_dir, target_dir, binary): diff --git a/deps/v8/tools/testrunner/local/fake_testsuite/fake_testsuite.status b/deps/v8/tools/testrunner/local/fake_testsuite/fake_testsuite.status new file mode 100644 index 0000000000..b5ebc84474 --- /dev/null +++ b/deps/v8/tools/testrunner/local/fake_testsuite/fake_testsuite.status @@ -0,0 +1,5 @@ +# Copyright 2019 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +[] diff --git a/deps/v8/tools/testrunner/local/fake_testsuite/testcfg.py b/deps/v8/tools/testrunner/local/fake_testsuite/testcfg.py new file mode 100644 index 0000000000..61d75fb991 --- /dev/null +++ b/deps/v8/tools/testrunner/local/fake_testsuite/testcfg.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# Copyright 2019 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import sys + +from testrunner.local import testsuite, statusfile + + +class TestSuite(testsuite.TestSuite): + def _test_class(self): + return testsuite.TestCase + + def ListTests(self): + fast = self._create_test("fast") + slow = self._create_test("slow") + slow._statusfile_outcomes.append(statusfile.SLOW) + yield fast + yield slow + +def GetSuite(*args, **kwargs): + return TestSuite(*args, **kwargs) diff --git a/deps/v8/tools/testrunner/local/statusfile.py b/deps/v8/tools/testrunner/local/statusfile.py index 4742e84caf..5d05e23cc3 100644 --- a/deps/v8/tools/testrunner/local/statusfile.py +++ b/deps/v8/tools/testrunner/local/statusfile.py @@ -45,12 +45,13 @@ FAIL_SLOPPY = "FAIL_SLOPPY" SKIP = "SKIP" SLOW = "SLOW" NO_VARIANTS = "NO_VARIANTS" +FAIL_PHASE_ONLY = "FAIL_PHASE_ONLY" ALWAYS = "ALWAYS" KEYWORDS = {} for key in [SKIP, FAIL, PASS, CRASH, SLOW, FAIL_OK, NO_VARIANTS, FAIL_SLOPPY, - ALWAYS]: + ALWAYS, FAIL_PHASE_ONLY]: KEYWORDS[key] = key # Support arches, modes to be written as keywords instead of strings. diff --git a/deps/v8/tools/testrunner/local/testsuite.py b/deps/v8/tools/testrunner/local/testsuite.py index 4bdfd008fe..9d00d7bc12 100644 --- a/deps/v8/tools/testrunner/local/testsuite.py +++ b/deps/v8/tools/testrunner/local/testsuite.py @@ -29,6 +29,7 @@ import fnmatch import imp import os +from contextlib import contextmanager from . import command from . import statusfile @@ -77,19 +78,22 @@ class TestCombiner(object): def _combined_test_class(self): raise NotImplementedError() +@contextmanager +def _load_testsuite_module(name, root): + f = None + try: + (f, pathname, description) = imp.find_module("testcfg", [root]) + yield imp.load_module(name + "_testcfg", f, pathname, description) + finally: + if f: + f.close() class TestSuite(object): @staticmethod - def LoadTestSuite(root, test_config): + def Load(root, test_config): name = root.split(os.path.sep)[-1] - f = None - try: - (f, pathname, description) = imp.find_module("testcfg", [root]) - module = imp.load_module(name + "_testcfg", f, pathname, description) + with _load_testsuite_module(name, root) as module: return module.GetSuite(name, root, test_config) - finally: - if f: - f.close() def __init__(self, name, root, test_config): self.name = name # string @@ -97,22 +101,22 @@ class TestSuite(object): self.test_config = test_config self.tests = None # list of TestCase objects self.statusfile = None - self.suppress_internals = False def status_file(self): return "%s/%s.status" % (self.root, self.name) - def do_suppress_internals(self): - """Specifies if this test suite should suppress asserts based on internals. - - Internals are e.g. testing against the outcome of native runtime functions. - This is switched off on some fuzzers that violate these contracts. - """ - self.suppress_internals = True - def ListTests(self): raise NotImplementedError + def load_tests_from_disk(self, statusfile_variables): + self.statusfile = statusfile.StatusFile( + self.status_file(), statusfile_variables) + + slow_tests = (test for test in self.ListTests() if test.is_slow) + fast_tests = (test for test in self.ListTests() if not test.is_slow) + + return slow_tests, fast_tests + def get_variants_gen(self, variants): return self._variants_gen_class()(variants) @@ -134,90 +138,11 @@ class TestSuite(object): """ return None - def ReadStatusFile(self, variables): - self.statusfile = statusfile.StatusFile(self.status_file(), variables) - - def ReadTestCases(self): - self.tests = self.ListTests() - - - def FilterTestCasesByStatus(self, - slow_tests_mode=None, - pass_fail_tests_mode=None): - """Filters tests by outcomes from status file. - - Status file has to be loaded before using this function. - - Args: - slow_tests_mode: What to do with slow tests. - pass_fail_tests_mode: What to do with pass or fail tests. - - Mode options: - None (default) - don't skip - "skip" - skip if slow/pass_fail - "run" - skip if not slow/pass_fail - """ - def _skip_slow(is_slow, mode): - return ( - (mode == 'run' and not is_slow) or - (mode == 'skip' and is_slow)) - - def _skip_pass_fail(pass_fail, mode): - return ( - (mode == 'run' and not pass_fail) or - (mode == 'skip' and pass_fail)) - - def _compliant(test): - if test.do_skip: - return False - if _skip_slow(test.is_slow, slow_tests_mode): - return False - if _skip_pass_fail(test.is_pass_or_fail, pass_fail_tests_mode): - return False - return True - - self.tests = filter(_compliant, self.tests) - - def FilterTestCasesByArgs(self, args): - """Filter test cases based on command-line arguments. - - args can be a glob: asterisks in any position of the argument - represent zero or more characters. Without asterisks, only exact matches - will be used with the exeption of the test-suite name as argument. - """ - filtered = [] - globs = [] - for a in args: - argpath = a.split('/') - if argpath[0] != self.name: - continue - if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'): - return # Don't filter, run all tests in this suite. - path = '/'.join(argpath[1:]) - globs.append(path) - - for t in self.tests: - for g in globs: - if fnmatch.fnmatch(t.path, g): - filtered.append(t) - break - self.tests = filtered - def _create_test(self, path, **kwargs): - if self.suppress_internals: - test_class = self._suppressed_test_class() - else: - test_class = self._test_class() + test_class = self._test_class() return test_class(self, path, self._path_to_name(path), self.test_config, **kwargs) - def _suppressed_test_class(self): - """Optional testcase that suppresses assertions. Used by fuzzers that are - only interested in dchecks or tsan and that might violate the assertions - through fuzzing. - """ - return self._test_class() - def _test_class(self): raise NotImplementedError diff --git a/deps/v8/tools/testrunner/local/testsuite_unittest.py b/deps/v8/tools/testrunner/local/testsuite_unittest.py index efefe4c533..efc9fdacf0 100755 --- a/deps/v8/tools/testrunner/local/testsuite_unittest.py +++ b/deps/v8/tools/testrunner/local/testsuite_unittest.py @@ -5,6 +5,7 @@ import os import sys +import tempfile import unittest # Needed because the test runner contains relative imports. @@ -14,107 +15,52 @@ sys.path.append(TOOLS_PATH) from testrunner.local.testsuite import TestSuite from testrunner.objects.testcase import TestCase +from testrunner.test_config import TestConfig class TestSuiteTest(unittest.TestCase): - def test_filter_testcases_by_status_first_pass(self): - suite = TestSuite('foo', 'bar') - suite.rules = { - '': { - 'foo/bar': set(['PASS', 'SKIP']), - 'baz/bar': set(['PASS', 'FAIL']), - }, - } - suite.prefix_rules = { - '': { - 'baz/': set(['PASS', 'SLOW']), - }, - } - suite.tests = [ - TestCase(suite, 'foo/bar', 'foo/bar'), - TestCase(suite, 'baz/bar', 'baz/bar'), - ] - suite.FilterTestCasesByStatus() - self.assertEquals( - [TestCase(suite, 'baz/bar', 'baz/bar')], - suite.tests, + def setUp(self): + test_dir = os.path.dirname(__file__) + self.test_root = os.path.join(test_dir, "fake_testsuite") + self.test_config = TestConfig( + command_prefix=[], + extra_flags=[], + isolates=False, + mode_flags=[], + no_harness=False, + noi18n=False, + random_seed=0, + run_skipped=False, + shell_dir='fake_testsuite/fake_d8', + timeout=10, + verbose=False, ) - outcomes = suite.GetStatusFileOutcomes(suite.tests[0].name, - suite.tests[0].variant) - self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), outcomes) - def test_filter_testcases_by_status_second_pass(self): - suite = TestSuite('foo', 'bar') + self.suite = TestSuite.Load(self.test_root, self.test_config) - suite.rules = { - '': { - 'foo/bar': set(['PREV']), - }, - 'default': { - 'foo/bar': set(['PASS', 'SKIP']), - 'baz/bar': set(['PASS', 'FAIL']), - }, - 'stress': { - 'baz/bar': set(['SKIP']), - }, - } - suite.prefix_rules = { - '': { - 'baz/': set(['PREV']), - }, - 'default': { - 'baz/': set(['PASS', 'SLOW']), - }, - 'stress': { - 'foo/': set(['PASS', 'SLOW']), - }, - } + def testLoadingTestSuites(self): + self.assertEquals(self.suite.name, "fake_testsuite") + self.assertEquals(self.suite.test_config, self.test_config) - test1 = TestCase(suite, 'foo/bar', 'foo/bar') - test2 = TestCase(suite, 'baz/bar', 'baz/bar') - suite.tests = [ - test1.create_variant(variant='default', flags=[]), - test1.create_variant(variant='stress', flags=['-v']), - test2.create_variant(variant='default', flags=[]), - test2.create_variant(variant='stress', flags=['-v']), - ] + # Verify that the components of the TestSuite aren't loaded yet. + self.assertIsNone(self.suite.tests) + self.assertIsNone(self.suite.statusfile) - suite.FilterTestCasesByStatus() - self.assertEquals( - [ - TestCase(suite, 'foo/bar', 'foo/bar').create_variant(None, ['-v']), - TestCase(suite, 'baz/bar', 'baz/bar'), - ], - suite.tests, - ) - - self.assertEquals( - set(['PREV', 'PASS', 'SLOW']), - suite.GetStatusFileOutcomes(suite.tests[0].name, - suite.tests[0].variant), - ) - self.assertEquals( - set(['PREV', 'PASS', 'FAIL', 'SLOW']), - suite.GetStatusFileOutcomes(suite.tests[1].name, - suite.tests[1].variant), - ) + def testLoadingTestsFromDisk(self): + slow_tests, fast_tests = self.suite.load_tests_from_disk( + statusfile_variables={}) + def is_generator(iterator): + return iterator == iter(iterator) - def test_fail_ok_outcome(self): - suite = TestSuite('foo', 'bar') - suite.rules = { - '': { - 'foo/bar': set(['FAIL_OK']), - 'baz/bar': set(['FAIL']), - }, - } - suite.prefix_rules = {} - suite.tests = [ - TestCase(suite, 'foo/bar', 'foo/bar'), - TestCase(suite, 'baz/bar', 'baz/bar'), - ] + self.assertTrue(is_generator(slow_tests)) + self.assertTrue(is_generator(fast_tests)) - for t in suite.tests: - self.assertEquals(['FAIL'], t.expected_outcomes) + slow_tests, fast_tests = list(slow_tests), list(fast_tests) + # Verify that the components of the TestSuite are loaded. + self.assertTrue(len(slow_tests) == len(fast_tests) == 1) + self.assertTrue(all(test.is_slow for test in slow_tests)) + self.assertFalse(any(test.is_slow for test in fast_tests)) + self.assertIsNotNone(self.suite.statusfile) if __name__ == '__main__': diff --git a/deps/v8/tools/testrunner/num_fuzzer.py b/deps/v8/tools/testrunner/num_fuzzer.py index 3b76541604..d5e399626c 100755 --- a/deps/v8/tools/testrunner/num_fuzzer.py +++ b/deps/v8/tools/testrunner/num_fuzzer.py @@ -20,7 +20,7 @@ from testrunner.testproc.execution import ExecutionProc from testrunner.testproc.expectation import ForgiveTimeoutProc from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc from testrunner.testproc.loader import LoadProc -from testrunner.testproc.progress import ResultsTracker, TestsCounter +from testrunner.testproc.progress import ResultsTracker from testrunner.utils import random_utils @@ -55,6 +55,11 @@ class NumFuzzer(base_runner.BaseTestRunner): parser.add_option("--stress-gc", default=0, type="int", help="probability [0-10] of adding --random-gc-interval " "flag to the test") + + # Stress tasks + parser.add_option("--stress-delay-tasks", default=0, type="int", + help="probability [0-10] of adding --stress-delay-tasks " + "flag to the test") parser.add_option("--stress-thread-pool-size", default=0, type="int", help="probability [0-10] of adding --thread-pool-size " "flag to the test") @@ -67,11 +72,6 @@ class NumFuzzer(base_runner.BaseTestRunner): help="extends --stress-deopt to have minimum interval " "between deopt points") - # Stress interrupt budget - parser.add_option("--stress-interrupt-budget", default=0, type="int", - help="probability [0-10] of adding --interrupt-budget " - "flag to the test") - # Combine multiple tests parser.add_option("--combine-tests", default=False, action="store_true", help="Combine multiple tests as one and run with " @@ -110,14 +110,6 @@ class NumFuzzer(base_runner.BaseTestRunner): def _get_default_suite_names(self): return DEFAULT_SUITES - def _timeout_scalefactor(self, options): - factor = super(NumFuzzer, self)._timeout_scalefactor(options) - if options.stress_interrupt_budget: - # TODO(machenbach): This should be moved to a more generic config. - # Fuzzers have too much timeout in debug mode. - factor = max(int(factor * 0.25), 1) - return factor - def _get_statusfile_variables(self, options): variables = ( super(NumFuzzer, self)._get_statusfile_variables(options)) @@ -129,6 +121,7 @@ class NumFuzzer(base_runner.BaseTestRunner): options.stress_scavenge, options.stress_compaction, options.stress_gc, + options.stress_delay_tasks, options.stress_thread_pool_size])), }) return variables @@ -180,15 +173,8 @@ class NumFuzzer(base_runner.BaseTestRunner): # Indicate if a SIGINT or SIGTERM happened. return sigproc.exit_code - def _load_suites(self, names, options): - suites = super(NumFuzzer, self)._load_suites(names, options) - if options.combine_tests: - suites = [s for s in suites if s.test_combiner_available()] - if options.stress_interrupt_budget: - # Changing interrupt budget forces us to suppress certain test assertions. - for suite in suites: - suite.do_suppress_internals() - return suites + def _is_testsuite_supported(self, suite, options): + return not options.combine_tests or suite.test_combiner_available() def _create_combiner(self, rng, options): if not options.combine_tests: @@ -211,7 +197,7 @@ class NumFuzzer(base_runner.BaseTestRunner): def _disable_analysis(self, options): """Disable analysis phase when options are used that don't support it.""" - return options.combine_tests or options.stress_interrupt_budget + return options.combine_tests def _create_fuzzer_configs(self, options): fuzzers = [] @@ -224,7 +210,7 @@ class NumFuzzer(base_runner.BaseTestRunner): add('scavenge', options.stress_scavenge) add('gc_interval', options.stress_gc) add('threads', options.stress_thread_pool_size) - add('interrupt_budget', options.stress_interrupt_budget) + add('delay', options.stress_delay_tasks) add('deopt', options.stress_deopt, options.stress_deopt_min) return fuzzers diff --git a/deps/v8/tools/testrunner/objects/testcase.py b/deps/v8/tools/testrunner/objects/testcase.py index de8bc561eb..c8f7bdbfb0 100644 --- a/deps/v8/tools/testrunner/objects/testcase.py +++ b/deps/v8/tools/testrunner/objects/testcase.py @@ -43,15 +43,15 @@ RESOURCES_PATTERN = re.compile(r"//\s+Resources:(.*)") # Pattern to auto-detect files to push on Android for statements like: # load("path/to/file.js") LOAD_PATTERN = re.compile( - r"(?:load|readbuffer|read)\((?:'|\")([^'\"]*)(?:'|\")\)") + r"(?:load|readbuffer|read)\((?:'|\")([^'\"]+)(?:'|\")\)") # Pattern to auto-detect files to push on Android for statements like: # import "path/to/file.js" MODULE_RESOURCES_PATTERN_1 = re.compile( - r"(?:import|export)(?:\(| )(?:'|\")([^'\"]*)(?:'|\")") + r"(?:import|export)(?:\(| )(?:'|\")([^'\"]+)(?:'|\")") # Pattern to auto-detect files to push on Android for statements like: # import foobar from "path/to/file.js" MODULE_RESOURCES_PATTERN_2 = re.compile( - r"(?:import|export).*from (?:'|\")([^'\"]*)(?:'|\")") + r"(?:import|export).*from (?:'|\")([^'\"]+)(?:'|\")") class TestCase(object): @@ -135,7 +135,8 @@ class TestCase(object): @property def do_skip(self): - return statusfile.SKIP in self._statusfile_outcomes + return (statusfile.SKIP in self._statusfile_outcomes and + not self.suite.test_config.run_skipped) @property def is_slow(self): diff --git a/deps/v8/tools/testrunner/outproc/test262.py b/deps/v8/tools/testrunner/outproc/test262.py index b5eb5547c3..bf3bc05809 100644 --- a/deps/v8/tools/testrunner/outproc/test262.py +++ b/deps/v8/tools/testrunner/outproc/test262.py @@ -7,18 +7,29 @@ import re from . import base +def _is_failure_output(output): + return ( + output.exit_code != 0 or + 'FAILED!' in output.stdout + ) + + class ExceptionOutProc(base.OutProc): """Output processor for tests with expected exception.""" - def __init__(self, expected_outcomes, expected_exception=None): + def __init__( + self, expected_outcomes, expected_exception=None, negative=False): super(ExceptionOutProc, self).__init__(expected_outcomes) self._expected_exception = expected_exception + self._negative = negative + + @property + def negative(self): + return self._negative def _is_failure_output(self, output): - if output.exit_code != 0: - return True if self._expected_exception != self._parse_exception(output.stdout): return True - return 'FAILED!' in output.stdout + return _is_failure_output(output) def _parse_exception(self, string): # somefile:somelinenumber: someerror[: sometext] @@ -31,16 +42,13 @@ class ExceptionOutProc(base.OutProc): return None -def _is_failure_output(self, output): - return ( - output.exit_code != 0 or - 'FAILED!' in output.stdout - ) - - class NoExceptionOutProc(base.OutProc): """Output processor optimized for tests without expected exception.""" -NoExceptionOutProc._is_failure_output = _is_failure_output + def __init__(self, expected_outcomes): + super(NoExceptionOutProc, self).__init__(expected_outcomes) + + def _is_failure_output(self, output): + return _is_failure_output(output) class PassNoExceptionOutProc(base.PassOutProc): @@ -48,7 +56,8 @@ class PassNoExceptionOutProc(base.PassOutProc): Output processor optimized for tests expected to PASS without expected exception. """ -PassNoExceptionOutProc._is_failure_output = _is_failure_output + def _is_failure_output(self, output): + return _is_failure_output(output) PASS_NO_EXCEPTION = PassNoExceptionOutProc() diff --git a/deps/v8/tools/testrunner/standard_runner.py b/deps/v8/tools/testrunner/standard_runner.py index bf7d3f133d..c84260c0a6 100755 --- a/deps/v8/tools/testrunner/standard_runner.py +++ b/deps/v8/tools/testrunner/standard_runner.py @@ -18,7 +18,7 @@ from testrunner.objects import predictable from testrunner.testproc.execution import ExecutionProc from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc from testrunner.testproc.loader import LoadProc -from testrunner.testproc.progress import ResultsTracker, TestsCounter +from testrunner.testproc.progress import ResultsTracker from testrunner.testproc.seed import SeedProc from testrunner.testproc.variant import VariantProc from testrunner.utils import random_utils @@ -49,7 +49,8 @@ VARIANT_ALIASES = { GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction", "--concurrent-recompilation-queue-length=64", "--concurrent-recompilation-delay=500", - "--concurrent-recompilation"] + "--concurrent-recompilation", + "--stress-flush-bytecode"] RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000", "--stress-compaction-random"] @@ -281,7 +282,6 @@ class StandardTestRunner(base_runner.BaseTestRunner): print '>>> Running with test processors' loader = LoadProc() - tests_counter = TestsCounter() results = self._create_result_tracker(options) indicators = self._create_progress_indicators(options) @@ -296,7 +296,6 @@ class StandardTestRunner(base_runner.BaseTestRunner): NameFilterProc(args) if args else None, StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), self._create_shard_proc(options), - tests_counter, VariantProc(self._variants), StatusFileFilterProc(options.slow_tests, options.pass_fail_tests), self._create_predictable_filter(), @@ -310,13 +309,9 @@ class StandardTestRunner(base_runner.BaseTestRunner): ] self._prepare_procs(procs) - tests.sort(key=lambda t: t.is_slow, reverse=True) loader.load_tests(tests) - print '>>> Running %d base tests' % tests_counter.total - tests_counter.remove_from_chain() - # This starts up worker processes and blocks until all tests are # processed. execproc.run() diff --git a/deps/v8/tools/testrunner/test_config.py b/deps/v8/tools/testrunner/test_config.py index d9418fe9ac..27ac72bf6c 100644 --- a/deps/v8/tools/testrunner/test_config.py +++ b/deps/v8/tools/testrunner/test_config.py @@ -16,6 +16,7 @@ class TestConfig(object): no_harness, noi18n, random_seed, + run_skipped, shell_dir, timeout, verbose): @@ -27,6 +28,7 @@ class TestConfig(object): self.noi18n = noi18n # random_seed is always not None. self.random_seed = random_seed or random_utils.random_seed() + self.run_skipped = run_skipped self.shell_dir = shell_dir self.timeout = timeout self.verbose = verbose diff --git a/deps/v8/tools/testrunner/testproc/fuzzer.py b/deps/v8/tools/testrunner/testproc/fuzzer.py index 624b9aac04..799b4bfb5e 100644 --- a/deps/v8/tools/testrunner/testproc/fuzzer.py +++ b/deps/v8/tools/testrunner/testproc/fuzzer.py @@ -217,17 +217,16 @@ class CompactionFuzzer(Fuzzer): yield ['--stress-compaction-random'] -class ThreadPoolSizeFuzzer(Fuzzer): +class TaskDelayFuzzer(Fuzzer): def create_flags_generator(self, rng, test, analysis_value): while True: - yield ['--thread-pool-size=%d' % rng.randint(1, 8)] + yield ['--stress-delay-tasks'] -class InterruptBudgetFuzzer(Fuzzer): +class ThreadPoolSizeFuzzer(Fuzzer): def create_flags_generator(self, rng, test, analysis_value): while True: - limit = 1 + int(rng.random() * 144) - yield ['--interrupt-budget=%d' % rng.randint(1, limit * 1024)] + yield ['--thread-pool-size=%d' % rng.randint(1, 8)] class DeoptAnalyzer(Analyzer): @@ -269,9 +268,9 @@ class DeoptFuzzer(Fuzzer): FUZZERS = { 'compaction': (None, CompactionFuzzer), + 'delay': (None, TaskDelayFuzzer), 'deopt': (DeoptAnalyzer, DeoptFuzzer), 'gc_interval': (GcIntervalAnalyzer, GcIntervalFuzzer), - 'interrupt_budget': (None, InterruptBudgetFuzzer), 'marking': (MarkingAnalyzer, MarkingFuzzer), 'scavenge': (ScavengeAnalyzer, ScavengeFuzzer), 'threads': (None, ThreadPoolSizeFuzzer), diff --git a/deps/v8/tools/testrunner/testproc/progress.py b/deps/v8/tools/testrunner/testproc/progress.py index 50b7307e1c..096228dc35 100644 --- a/deps/v8/tools/testrunner/testproc/progress.py +++ b/deps/v8/tools/testrunner/testproc/progress.py @@ -22,15 +22,6 @@ def print_failure_header(test): } -class TestsCounter(base.TestProcObserver): - def __init__(self): - super(TestsCounter, self).__init__() - self.total = 0 - - def _on_next_test(self, test): - self.total += 1 - - class ResultsTracker(base.TestProcObserver): """Tracks number of results and stops to run tests if max_failures reached.""" def __init__(self, max_failures): @@ -66,10 +57,6 @@ class SimpleProgressIndicator(ProgressIndicator): self._requirement = base.DROP_PASS_OUTPUT self._failed = [] - self._total = 0 - - def _on_next_test(self, test): - self._total += 1 def _on_result_for(self, test, result): # TODO(majeski): Support for dummy/grouped results @@ -170,13 +157,9 @@ class CompactProgressIndicator(ProgressIndicator): self._last_status_length = 0 self._start_time = time.time() - self._total = 0 self._passed = 0 self._failed = 0 - def _on_next_test(self, test): - self._total += 1 - def _on_result_for(self, test, result): # TODO(majeski): Support for dummy/grouped results if result.has_unexpected_output: @@ -210,13 +193,8 @@ class CompactProgressIndicator(ProgressIndicator): def _print_progress(self, name): self._clear_line(self._last_status_length) elapsed = time.time() - self._start_time - if not self._total: - progress = 0 - else: - progress = (self._passed + self._failed) * 100 // self._total status = self._templates['status_line'] % { 'passed': self._passed, - 'progress': progress, 'failed': self._failed, 'test': name, 'mins': int(elapsed) / 60, @@ -241,7 +219,6 @@ class ColorProgressIndicator(CompactProgressIndicator): def __init__(self): templates = { 'status_line': ("[%(mins)02i:%(secs)02i|" - "\033[34m%%%(progress) 4d\033[0m|" "\033[32m+%(passed) 4d\033[0m|" "\033[31m-%(failed) 4d\033[0m]: %(test)s"), 'stdout': "\033[1m%s\033[0m", @@ -256,7 +233,7 @@ class ColorProgressIndicator(CompactProgressIndicator): class MonochromeProgressIndicator(CompactProgressIndicator): def __init__(self): templates = { - 'status_line': ("[%(mins)02i:%(secs)02i|%%%(progress) 4d|" + 'status_line': ("[%(mins)02i:%(secs)02i|" "+%(passed) 4d|-%(failed) 4d]: %(test)s"), 'stdout': '%s', 'stderr': '%s', |