From a9812142ca4116b425e92274c437d99e1d713fe2 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 20 Dec 2018 14:06:15 +0100 Subject: deps: patch V8 to 7.1.302.33 PR-URL: https://github.com/nodejs/node/pull/25101 Refs: https://github.com/v8/v8/compare/7.1.302.28...7.1.302.33 Fixes: https://github.com/nodejs/node/issues/25089 Reviewed-By: Richard Lau Reviewed-By: Gus Caplan Reviewed-By: James M Snell Reviewed-By: Ali Ijaz Sheikh Reviewed-By: Yang Guo --- deps/v8/DEPS | 25 +++- deps/v8/include/v8-version.h | 2 +- deps/v8/src/builtins/array-splice.tq | 103 +++++++------ deps/v8/src/builtins/base.tq | 32 ++++ deps/v8/src/code-stub-assembler.cc | 173 ++++++++++++++++++++++ deps/v8/src/code-stub-assembler.h | 23 +++ deps/v8/src/compiler/representation-change.cc | 2 +- deps/v8/test/mjsunit/regress/regress-895691.js | 18 +++ deps/v8/tools/__init__.py | 4 + deps/v8/tools/unittests/__init__.py | 4 + deps/v8/tools/unittests/v8_presubmit_test.py | 91 ++++++++++++ deps/v8/tools/v8_presubmit.py | 194 ++++++++++++++----------- 12 files changed, 544 insertions(+), 127 deletions(-) create mode 100644 deps/v8/test/mjsunit/regress/regress-895691.js create mode 100644 deps/v8/tools/__init__.py create mode 100644 deps/v8/tools/unittests/__init__.py create mode 100755 deps/v8/tools/unittests/v8_presubmit_test.py (limited to 'deps') diff --git a/deps/v8/DEPS b/deps/v8/DEPS index fdd96f9b82..a87c01e49d 100644 --- a/deps/v8/DEPS +++ b/deps/v8/DEPS @@ -9,6 +9,9 @@ vars = { 'download_jsfunfuzz': False, 'download_mips_toolchain': False, 'check_v8_header_includes': False, + + # luci-go CIPD package version. + 'luci_go': 'git_revision:fdf05508e8a66c773a41521e0243c9d11b9a2a1c', } deps = { @@ -51,7 +54,7 @@ deps = { 'v8/third_party/markupsafe': Var('chromium_url') + '/chromium/src/third_party/markupsafe.git' + '@' + '8f45f5cfa0009d2a70589bcda0349b8cb2b72783', 'v8/tools/swarming_client': - Var('chromium_url') + '/infra/luci/client-py.git' + '@' + '486c9b53c4d54dd4b95bb6ce0e31160e600dfc11', + Var('chromium_url') + '/infra/luci/client-py.git' + '@' + '0e3e1c4dc4e79f25a5b58fcbc135dc93183c0c54', 'v8/test/benchmarks/data': Var('chromium_url') + '/v8/deps/third_party/benchmarks.git' + '@' + '05d7188267b4560491ff9155c5ee13e207ecd65f', 'v8/test/mozilla/data': @@ -82,8 +85,24 @@ deps = { }, 'v8/tools/clang': Var('chromium_url') + '/chromium/src/tools/clang.git' + '@' + '7792d28b069af6dd3a86d1ba83b7f5c4ede605dc', - 'v8/tools/luci-go': - Var('chromium_url') + '/chromium/src/tools/luci-go.git' + '@' + '445d7c4b6a4f10e188edb395b132e3996b127691', + 'v8/tools/luci-go': { + 'packages': [ + { + 'package': 'infra/tools/luci/isolate/${{platform}}', + 'version': Var('luci_go'), + }, + { + 'package': 'infra/tools/luci/isolated/${{platform}}', + 'version': Var('luci_go'), + }, + { + 'package': 'infra/tools/luci/swarming/${{platform}}', + 'version': Var('luci_go'), + }, + ], + 'condition': 'host_cpu != "s390"', + 'dep_type': 'cipd', + }, 'v8/test/wasm-js': Var('chromium_url') + '/external/github.com/WebAssembly/spec.git' + '@' + 'db9cd40808a90ecc5f4a23e88fb375c8f60b8d52', } diff --git a/deps/v8/include/v8-version.h b/deps/v8/include/v8-version.h index 8624767047..114e57c58e 100644 --- a/deps/v8/include/v8-version.h +++ b/deps/v8/include/v8-version.h @@ -11,7 +11,7 @@ #define V8_MAJOR_VERSION 7 #define V8_MINOR_VERSION 1 #define V8_BUILD_NUMBER 302 -#define V8_PATCH_LEVEL 28 +#define V8_PATCH_LEVEL 33 // Use 1 for candidates and 0 otherwise. // (Boolean macro values are not supported by all preprocessors.) diff --git a/deps/v8/src/builtins/array-splice.tq b/deps/v8/src/builtins/array-splice.tq index 16a192d2c0..5746f4cdf6 100644 --- a/deps/v8/src/builtins/array-splice.tq +++ b/deps/v8/src/builtins/array-splice.tq @@ -9,8 +9,12 @@ module array { // zero-length input FixedArray is handled here. macro Extract( elements: FixedArrayBase, first: Smi, count: Smi, - capacity: Smi): FixedArrayType { - return UnsafeCast( + capacity: Smi): FixedArrayType; + + Extract( + elements: FixedArrayBase, first: Smi, count: Smi, + capacity: Smi): FixedArray { + return UnsafeCast( ExtractFixedArray(elements, first, count, capacity)); } @@ -24,31 +28,73 @@ module array { ExtractFixedArray(elements, first, count, capacity)); } + macro DoMoveElements( + elements: FixedArrayType, dstIndex: Smi, srcIndex: Smi, + count: Smi): void { + TorqueMoveElements( + elements, Convert(dstIndex), Convert(srcIndex), + Convert(count)); + } + + macro StoreHoles( + elements: FixedArrayType, holeStartIndex: Smi, holeEndIndex: Smi): void { + for (let i: Smi = holeStartIndex; i < holeEndIndex; i++) { + StoreArrayHole(elements, i); + } + } + + macro DoCopyElements( + dstElements: FixedArrayType, dstIndex: Smi, srcElements: FixedArrayType, + srcIndex: Smi, count: Smi): void { + TorqueCopyElements( + dstElements, Convert(dstIndex), srcElements, + Convert(srcIndex), Convert(count)); + } + macro FastSplice( args: constexpr Arguments, a: JSArray, length: Smi, newLength: Smi, lengthDelta: Smi, actualStart: Smi, insertCount: Smi, actualDeleteCount: Smi): void labels Bailout { - const elements: FixedArrayBase = a.elements; - const elementsMap: Map = elements.map; - - // If the spliced array is larger then the - // source array, then allocate a new FixedArrayType to hold the result. - let newElements: FixedArrayBase = elements; - if (elementsMap == kCOWMap || lengthDelta > 0) { - newElements = - Extract(elements, 0, actualStart, newLength); - if (elementsMap == kCOWMap) { - newElements.map = elementsMap; + // Make sure elements are writable. + EnsureWriteableFastElements(a); + + if (insertCount != actualDeleteCount) { + const elements: FixedArrayBase = a.elements; + const dstIndex: Smi = actualStart + insertCount; + const srcIndex: Smi = actualStart + actualDeleteCount; + const count: Smi = length - actualDeleteCount - actualStart; + if (insertCount < actualDeleteCount) { + // Shrink. + DoMoveElements( + UnsafeCast(elements), dstIndex, srcIndex, count); + StoreHoles( + UnsafeCast(elements), newLength, length); + } else if (insertCount > actualDeleteCount) { + // If the backing store is big enough, then moving elements is enough. + if (newLength <= elements.length) { + DoMoveElements( + UnsafeCast(elements), dstIndex, srcIndex, count); + } else { + // Grow. + let capacity: Smi = CalculateNewElementsCapacity(newLength); + const newElements: FixedArrayType = + Extract(elements, 0, actualStart, capacity); + a.elements = newElements; + if (elements.length > 0) { + DoCopyElements( + newElements, dstIndex, UnsafeCast(elements), + srcIndex, count); + } + } } - a.elements = newElements; } - // Copy over inserted elements. + // Copy arguments. let k: Smi = actualStart; if (insertCount > 0) { const typedNewElements: FixedArrayType = - UnsafeCast(newElements); + UnsafeCast(a.elements); for (let e: Object of args [2: ]) { // The argument elements were already validated to be an appropriate // {ElementType} to store in {FixedArrayType}. @@ -56,31 +102,6 @@ module array { } } - // Copy over elements after deleted elements. - let count: Smi = length - actualStart - actualDeleteCount; - while (count > 0) { - const typedElements: FixedArrayType = - UnsafeCast(elements); - const typedNewElements: FixedArrayType = - UnsafeCast(newElements); - CopyArrayElement(typedElements, typedNewElements, k - lengthDelta, k); - k++; - count--; - } - - // Fill rest of spliced FixedArray with the hole, but only if the - // destination FixedArray is the original array's, since otherwise the array - // is pre-filled with holes. - if (elements == newElements) { - const typedNewElements: FixedArrayType = - UnsafeCast(newElements); - const limit: Smi = elements.length; - while (k < limit) { - StoreArrayHole(typedNewElements, k); - k++; - } - } - // Update the array's length after all the FixedArray shuffling is done. a.length = newLength; } diff --git a/deps/v8/src/builtins/base.tq b/deps/v8/src/builtins/base.tq index 20c4f4b9e0..3f5029834d 100644 --- a/deps/v8/src/builtins/base.tq +++ b/deps/v8/src/builtins/base.tq @@ -844,6 +844,8 @@ macro AllowNonNumberElements(kind: ElementsKind): ElementsKind { extern macro AllocateZeroedFixedArray(intptr): FixedArray; extern macro AllocateZeroedFixedDoubleArray(intptr): FixedDoubleArray; +extern macro CalculateNewElementsCapacity(Smi): Smi; + extern macro CopyFixedArrayElements( constexpr ElementsKind, FixedArray, constexpr ElementsKind, FixedArray, intptr, intptr, intptr): void; @@ -879,6 +881,36 @@ extern macro ExtractFixedArray( extern builtin ExtractFastJSArray(Context, JSArray, Smi, Smi): JSArray; +extern macro MoveElements( + constexpr ElementsKind, FixedArrayBase, intptr, intptr, intptr): void; +macro TorqueMoveElements( + elements: FixedArray, dstIndex: intptr, srcIndex: intptr, + count: intptr): void { + MoveElements(HOLEY_ELEMENTS, elements, dstIndex, srcIndex, count); +} +macro TorqueMoveElements( + elements: FixedDoubleArray, dstIndex: intptr, srcIndex: intptr, + count: intptr): void { + MoveElements(HOLEY_DOUBLE_ELEMENTS, elements, dstIndex, srcIndex, count); +} + +extern macro CopyElements( + constexpr ElementsKind, FixedArrayBase, intptr, FixedArrayBase, intptr, + intptr): void; +macro TorqueCopyElements( + dstElements: FixedArray, dstIndex: intptr, srcElements: FixedArray, + srcIndex: intptr, count: intptr): void { + CopyElements( + HOLEY_ELEMENTS, dstElements, dstIndex, srcElements, srcIndex, count); +} +macro TorqueCopyElements( + dstElements: FixedDoubleArray, dstIndex: intptr, + srcElements: FixedDoubleArray, srcIndex: intptr, count: intptr): void { + CopyElements( + HOLEY_DOUBLE_ELEMENTS, dstElements, dstIndex, srcElements, srcIndex, + count); +} + macro LoadElementNoHole(a: JSArray, index: Smi): Object labels IfHole; diff --git a/deps/v8/src/code-stub-assembler.cc b/deps/v8/src/code-stub-assembler.cc index e307ca5cc3..1474e3d97d 100644 --- a/deps/v8/src/code-stub-assembler.cc +++ b/deps/v8/src/code-stub-assembler.cc @@ -4543,6 +4543,179 @@ void CodeStubAssembler::FillFixedDoubleArrayWithZero( backing_store, IntPtrConstant(0), byte_length); } +void CodeStubAssembler::JumpIfPointersFromHereAreInteresting( + TNode object, Label* interesting) { + Label finished(this); + TNode object_word = BitcastTaggedToWord(object); + TNode object_page = PageFromAddress(object_word); + TNode page_flags = UncheckedCast(Load( + MachineType::IntPtr(), object_page, IntPtrConstant(Page::kFlagsOffset))); + Branch( + WordEqual(WordAnd(page_flags, + IntPtrConstant( + MemoryChunk::kPointersFromHereAreInterestingMask)), + IntPtrConstant(0)), + &finished, interesting); + BIND(&finished); +} + +void CodeStubAssembler::MoveElements(ElementsKind kind, + TNode elements, + TNode dst_index, + TNode src_index, + TNode length) { + Label finished(this); + Label needs_barrier(this); + const bool needs_barrier_check = IsObjectElementsKind(kind); + + DCHECK(IsFastElementsKind(kind)); + CSA_ASSERT(this, IsFixedArrayWithKind(elements, kind)); + CSA_ASSERT(this, + IntPtrLessThanOrEqual(IntPtrAdd(dst_index, length), + LoadAndUntagFixedArrayBaseLength(elements))); + CSA_ASSERT(this, + IntPtrLessThanOrEqual(IntPtrAdd(src_index, length), + LoadAndUntagFixedArrayBaseLength(elements))); + + // The write barrier can be ignored if {elements} is in new space, or if + // we have a SMI or double ElementsKind. + if (needs_barrier_check) { + JumpIfPointersFromHereAreInteresting(elements, &needs_barrier); + } + + const TNode source_byte_length = + IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind))); + static const int32_t fa_base_data_offset = + FixedArrayBase::kHeaderSize - kHeapObjectTag; + TNode elements_intptr = BitcastTaggedToWord(elements); + TNode target_data_ptr = + IntPtrAdd(elements_intptr, + ElementOffsetFromIndex(dst_index, kind, INTPTR_PARAMETERS, + fa_base_data_offset)); + TNode source_data_ptr = + IntPtrAdd(elements_intptr, + ElementOffsetFromIndex(src_index, kind, INTPTR_PARAMETERS, + fa_base_data_offset)); + TNode memmove = + ExternalConstant(ExternalReference::libc_memmove_function()); + CallCFunction3(MachineType::Pointer(), MachineType::Pointer(), + MachineType::Pointer(), MachineType::UintPtr(), memmove, + target_data_ptr, source_data_ptr, source_byte_length); + + if (needs_barrier_check) { + Goto(&finished); + + BIND(&needs_barrier); + { + const TNode begin = src_index; + const TNode end = IntPtrAdd(begin, length); + + // If dst_index is less than src_index, then walk forward. + const TNode delta = + IntPtrMul(IntPtrSub(dst_index, begin), + IntPtrConstant(ElementsKindToByteSize(kind))); + auto loop_body = [&](Node* array, Node* offset) { + Node* const element = Load(MachineType::AnyTagged(), array, offset); + Node* const delta_offset = IntPtrAdd(offset, delta); + Store(array, delta_offset, element); + }; + + Label iterate_forward(this); + Label iterate_backward(this); + Branch(IntPtrLessThan(delta, IntPtrConstant(0)), &iterate_forward, + &iterate_backward); + BIND(&iterate_forward); + { + // Make a loop for the stores. + BuildFastFixedArrayForEach(elements, kind, begin, end, loop_body, + INTPTR_PARAMETERS, + ForEachDirection::kForward); + Goto(&finished); + } + + BIND(&iterate_backward); + { + BuildFastFixedArrayForEach(elements, kind, begin, end, loop_body, + INTPTR_PARAMETERS, + ForEachDirection::kReverse); + Goto(&finished); + } + } + BIND(&finished); + } +} + +void CodeStubAssembler::CopyElements(ElementsKind kind, + TNode dst_elements, + TNode dst_index, + TNode src_elements, + TNode src_index, + TNode length) { + Label finished(this); + Label needs_barrier(this); + const bool needs_barrier_check = IsObjectElementsKind(kind); + + DCHECK(IsFastElementsKind(kind)); + CSA_ASSERT(this, IsFixedArrayWithKind(dst_elements, kind)); + CSA_ASSERT(this, IsFixedArrayWithKind(src_elements, kind)); + CSA_ASSERT(this, IntPtrLessThanOrEqual( + IntPtrAdd(dst_index, length), + LoadAndUntagFixedArrayBaseLength(dst_elements))); + CSA_ASSERT(this, IntPtrLessThanOrEqual( + IntPtrAdd(src_index, length), + LoadAndUntagFixedArrayBaseLength(src_elements))); + CSA_ASSERT(this, WordNotEqual(dst_elements, src_elements)); + + // The write barrier can be ignored if {dst_elements} is in new space, or if + // we have a SMI or double ElementsKind. + if (needs_barrier_check) { + JumpIfPointersFromHereAreInteresting(dst_elements, &needs_barrier); + } + + TNode source_byte_length = + IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind))); + static const int32_t fa_base_data_offset = + FixedArrayBase::kHeaderSize - kHeapObjectTag; + TNode src_offset_start = ElementOffsetFromIndex( + src_index, kind, INTPTR_PARAMETERS, fa_base_data_offset); + TNode dst_offset_start = ElementOffsetFromIndex( + dst_index, kind, INTPTR_PARAMETERS, fa_base_data_offset); + TNode src_elements_intptr = BitcastTaggedToWord(src_elements); + TNode source_data_ptr = + IntPtrAdd(src_elements_intptr, src_offset_start); + TNode dst_elements_intptr = BitcastTaggedToWord(dst_elements); + TNode dst_data_ptr = + IntPtrAdd(dst_elements_intptr, dst_offset_start); + TNode memcpy = + ExternalConstant(ExternalReference::libc_memcpy_function()); + CallCFunction3(MachineType::Pointer(), MachineType::Pointer(), + MachineType::Pointer(), MachineType::UintPtr(), memcpy, + dst_data_ptr, source_data_ptr, source_byte_length); + + if (needs_barrier_check) { + Goto(&finished); + + BIND(&needs_barrier); + { + const TNode begin = src_index; + const TNode end = IntPtrAdd(begin, length); + const TNode delta = + IntPtrMul(IntPtrSub(dst_index, src_index), + IntPtrConstant(ElementsKindToByteSize(kind))); + BuildFastFixedArrayForEach( + src_elements, kind, begin, end, + [&](Node* array, Node* offset) { + Node* const element = Load(MachineType::AnyTagged(), array, offset); + Node* const delta_offset = IntPtrAdd(offset, delta); + Store(dst_elements, delta_offset, element); + }, + INTPTR_PARAMETERS, ForEachDirection::kForward); + Goto(&finished); + } + BIND(&finished); + } +} + void CodeStubAssembler::CopyFixedArrayElements( ElementsKind from_kind, Node* from_array, ElementsKind to_kind, Node* to_array, Node* first_element, Node* element_count, Node* capacity, diff --git a/deps/v8/src/code-stub-assembler.h b/deps/v8/src/code-stub-assembler.h index 69ac5e27bb..8bd39369b7 100644 --- a/deps/v8/src/code-stub-assembler.h +++ b/deps/v8/src/code-stub-assembler.h @@ -1567,6 +1567,25 @@ class V8_EXPORT_PRIVATE CodeStubAssembler : public compiler::CodeAssembler { SMI_PARAMETERS); } + void JumpIfPointersFromHereAreInteresting(TNode object, + Label* interesting); + + // Efficiently copy elements within a single array. The regions + // [src_index, src_index + length) and [dst_index, dst_index + length) + // can be overlapping. + void MoveElements(ElementsKind kind, TNode elements, + TNode dst_index, TNode src_index, + TNode length); + + // Efficiently copy elements from one array to another. The ElementsKind + // needs to be the same. Copy from src_elements at + // [src_index, src_index + length) to dst_elements at + // [dst_index, dst_index + length). + void CopyElements(ElementsKind kind, TNode dst_elements, + TNode dst_index, + TNode src_elements, + TNode src_index, TNode length); + TNode HeapObjectToFixedArray(TNode base, Label* cast_fail); @@ -1740,6 +1759,10 @@ class V8_EXPORT_PRIVATE CodeStubAssembler : public compiler::CodeAssembler { Node* CalculateNewElementsCapacity(Node* old_capacity, ParameterMode mode = INTPTR_PARAMETERS); + TNode CalculateNewElementsCapacity(TNode old_capacity) { + return CAST(CalculateNewElementsCapacity(old_capacity, SMI_PARAMETERS)); + } + // Tries to grow the |elements| array of given |object| to store the |key| // or bails out if the growing gap is too big. Returns new elements. Node* TryGrowElementsCapacity(Node* object, Node* elements, ElementsKind kind, diff --git a/deps/v8/src/compiler/representation-change.cc b/deps/v8/src/compiler/representation-change.cc index ad4c5c916c..b141cad773 100644 --- a/deps/v8/src/compiler/representation-change.cc +++ b/deps/v8/src/compiler/representation-change.cc @@ -586,7 +586,7 @@ Node* RepresentationChanger::GetFloat32RepresentationFor( } else if (output_rep == MachineRepresentation::kFloat64) { op = machine()->TruncateFloat64ToFloat32(); } else if (output_rep == MachineRepresentation::kWord64) { - if (output_type.Is(Type::Signed32())) { + if (output_type.Is(cache_.kSafeInteger)) { // int64 -> float64 -> float32 op = machine()->ChangeInt64ToFloat64(); node = jsgraph()->graph()->NewNode(op, node); diff --git a/deps/v8/test/mjsunit/regress/regress-895691.js b/deps/v8/test/mjsunit/regress/regress-895691.js new file mode 100644 index 0000000000..6f63ac6c9b --- /dev/null +++ b/deps/v8/test/mjsunit/regress/regress-895691.js @@ -0,0 +1,18 @@ +// Copyright 2018 the V8 project authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Flags: --allow-natives-syntax + +const n = 2**32; +const x = new Float32Array(); + +function f() { + for (var i = 96; i < 100; i += 4) { + x[i] = i + n; + } +} + +f(); +%OptimizeFunctionOnNextCall(f); +f(); diff --git a/deps/v8/tools/__init__.py b/deps/v8/tools/__init__.py new file mode 100644 index 0000000000..3841a861c8 --- /dev/null +++ b/deps/v8/tools/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/deps/v8/tools/unittests/__init__.py b/deps/v8/tools/unittests/__init__.py new file mode 100644 index 0000000000..3841a861c8 --- /dev/null +++ b/deps/v8/tools/unittests/__init__.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. diff --git a/deps/v8/tools/unittests/v8_presubmit_test.py b/deps/v8/tools/unittests/v8_presubmit_test.py new file mode 100755 index 0000000000..2c66d1891b --- /dev/null +++ b/deps/v8/tools/unittests/v8_presubmit_test.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python +# Copyright 2018 the V8 project authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +import os +import sys +import tempfile +import unittest + +# Configuring the path for the v8_presubmit module +TOOLS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.append(TOOLS_ROOT) + +from v8_presubmit import FileContentsCache, CacheableSourceFileProcessor + + +class FakeCachedProcessor(CacheableSourceFileProcessor): + def __init__(self, cache_file_path): + super(FakeCachedProcessor, self).__init__( + use_cache=True, cache_file_path=cache_file_path, file_type='.test') + def GetProcessorWorker(self): + return object + def GetProcessorScript(self): + return "echo", [] + def DetectUnformattedFiles(_, cmd, worker, files): + raise NotImplementedError + +class FileContentsCacheTest(unittest.TestCase): + def setUp(self): + _, self.cache_file_path = tempfile.mkstemp() + cache = FileContentsCache(self.cache_file_path) + cache.Load() + + def generate_file(): + _, file_name = tempfile.mkstemp() + with open(file_name, "w") as f: + f.write(file_name) + + return file_name + + self.target_files = [generate_file() for _ in range(2)] + unchanged_files = cache.FilterUnchangedFiles(self.target_files) + self.assertEqual(len(unchanged_files), 2) + cache.Save() + + def tearDown(self): + for file in [self.cache_file_path] + self.target_files: + os.remove(file) + + def testCachesFiles(self): + cache = FileContentsCache(self.cache_file_path) + cache.Load() + + changed_files = cache.FilterUnchangedFiles(self.target_files) + self.assertListEqual(changed_files, []) + + modified_file = self.target_files[0] + with open(modified_file, "w") as f: + f.write("modification") + + changed_files = cache.FilterUnchangedFiles(self.target_files) + self.assertListEqual(changed_files, [modified_file]) + + def testCacheableSourceFileProcessor(self): + class CachedProcessor(FakeCachedProcessor): + def DetectFilesToChange(_, files): + self.assertListEqual(files, []) + return [] + + cached_processor = CachedProcessor(cache_file_path=self.cache_file_path) + cached_processor.ProcessFiles(self.target_files) + + def testCacheableSourceFileProcessorWithModifications(self): + modified_file = self.target_files[0] + with open(modified_file, "w") as f: + f.write("modification") + + class CachedProcessor(FakeCachedProcessor): + def DetectFilesToChange(_, files): + self.assertListEqual(files, [modified_file]) + return [] + + cached_processor = CachedProcessor( + cache_file_path=self.cache_file_path, + ) + cached_processor.ProcessFiles(self.target_files) + + +if __name__ == '__main__': + unittest.main() diff --git a/deps/v8/tools/v8_presubmit.py b/deps/v8/tools/v8_presubmit.py index f35bd9a2ee..22d6bf5389 100755 --- a/deps/v8/tools/v8_presubmit.py +++ b/deps/v8/tools/v8_presubmit.py @@ -228,17 +228,98 @@ class SourceFileProcessor(object): return result -class CppLintProcessor(SourceFileProcessor): +class CacheableSourceFileProcessor(SourceFileProcessor): + """Utility class that allows caching ProcessFiles() method calls. + + In order to use it, create a ProcessFilesWithoutCaching method that returns + the files requiring intervention after processing the source files. + """ + + def __init__(self, use_cache, cache_file_path, file_type): + self.use_cache = use_cache + self.cache_file_path = cache_file_path + self.file_type = file_type + + def GetProcessorWorker(self): + """Expected to return the worker function to run the formatter.""" + raise NotImplementedError + + def GetProcessorScript(self): + """Expected to return a tuple + (path to the format processor script, list of arguments).""" + raise NotImplementedError + + def GetProcessorCommand(self): + format_processor, options = self.GetProcessorScript() + if not format_processor: + print('Could not find the formatter for % files' % self.file_type) + sys.exit(1) + + command = [sys.executable, format_processor] + command.extend(options) + + return command + + def ProcessFiles(self, files): + if self.use_cache: + cache = FileContentsCache(self.cache_file_path) + cache.Load() + files = cache.FilterUnchangedFiles(files) + + if len(files) == 0: + print 'No changes in %s files detected. Skipping check' % self.file_type + return True + + files_requiring_changes = self.DetectFilesToChange(files) + print ( + 'Total %s files found that require formatting: %d' % + (self.file_type, len(files_requiring_changes))) + if self.use_cache: + for file in files_requiring_changes: + cache.RemoveFile(file) + + cache.Save() + + return files_requiring_changes == [] + + def DetectFilesToChange(self, files): + command = self.GetProcessorCommand() + worker = self.GetProcessorWorker() + + commands = [command + [file] for file in files] + count = multiprocessing.cpu_count() + pool = multiprocessing.Pool(count) + try: + results = pool.map_async(worker, commands).get(timeout=240) + except KeyboardInterrupt: + print "\nCaught KeyboardInterrupt, terminating workers." + pool.terminate() + pool.join() + sys.exit(1) + + unformatted_files = [] + for index, errors in enumerate(results): + if errors > 0: + unformatted_files.append(files[index]) + + return unformatted_files + + +class CppLintProcessor(CacheableSourceFileProcessor): """ Lint files to check that they follow the google code style. """ + def __init__(self, use_cache=True): + super(CppLintProcessor, self).__init__( + use_cache=use_cache, cache_file_path='.cpplint-cache', file_type='C/C++') + def IsRelevant(self, name): return name.endswith('.cc') or name.endswith('.h') def IgnoreDir(self, name): return (super(CppLintProcessor, self).IgnoreDir(name) - or (name == 'third_party')) + or (name == 'third_party')) IGNORE_LINT = ['export-template.h', 'flag-definitions.h'] @@ -251,55 +332,30 @@ class CppLintProcessor(SourceFileProcessor): test_dirs = ['cctest', 'common', 'fuzzer', 'inspector', 'unittests'] return dirs + [join('test', dir) for dir in test_dirs] - def GetCpplintScript(self, prio_path): - for path in [prio_path] + os.environ["PATH"].split(os.pathsep): + def GetProcessorWorker(self): + return CppLintWorker + + def GetProcessorScript(self): + filters = ','.join([n for n in LINT_RULES]) + arguments = ['--filter', filters] + for path in [TOOLS_PATH] + os.environ["PATH"].split(os.pathsep): path = path.strip('"') - cpplint = os.path.join(path, "cpplint.py") + cpplint = os.path.join(path, 'cpplint.py') if os.path.isfile(cpplint): - return cpplint - - return None - - def ProcessFiles(self, files): - good_files_cache = FileContentsCache('.cpplint-cache') - good_files_cache.Load() - files = good_files_cache.FilterUnchangedFiles(files) - if len(files) == 0: - print 'No changes in C/C++ files detected. Skipping cpplint check.' - return True - - filters = ",".join([n for n in LINT_RULES]) - cpplint = self.GetCpplintScript(TOOLS_PATH) - if cpplint is None: - print('Could not find cpplint.py. Make sure ' - 'depot_tools is installed and in the path.') - sys.exit(1) - - command = [sys.executable, cpplint, '--filter', filters] - - commands = [command + [file] for file in files] - count = multiprocessing.cpu_count() - pool = multiprocessing.Pool(count) - try: - results = pool.map_async(CppLintWorker, commands).get(999999) - except KeyboardInterrupt: - print "\nCaught KeyboardInterrupt, terminating workers." - sys.exit(1) + return cpplint, arguments - for i in range(len(files)): - if results[i] > 0: - good_files_cache.RemoveFile(files[i]) + return None, arguments - total_errors = sum(results) - print "Total C/C++ files found that require formatting: %d" % total_errors - good_files_cache.Save() - return total_errors == 0 -class TorqueFormatProcessor(SourceFileProcessor): +class TorqueFormatProcessor(CacheableSourceFileProcessor): """ Check .tq files to verify they follow the Torque style guide. """ + def __init__(self, use_cache=True): + super(TorqueFormatProcessor, self).__init__( + use_cache=use_cache, cache_file_path='.torquelint-cache', file_type='Torque') + def IsRelevant(self, name): return name.endswith('.tq') @@ -308,47 +364,17 @@ class TorqueFormatProcessor(SourceFileProcessor): test_dirs = ['torque'] return dirs + [join('test', dir) for dir in test_dirs] - def GetTorquelintScript(self): + def GetProcessorWorker(self): + return TorqueLintWorker + + def GetProcessorScript(self): torque_tools = os.path.join(TOOLS_PATH, "torque") torque_path = os.path.join(torque_tools, "format-torque.py") - + arguments = ['-l'] if os.path.isfile(torque_path): - return torque_path - - return None - - def ProcessFiles(self, files): - good_files_cache = FileContentsCache('.torquelint-cache') - good_files_cache.Load() - files = good_files_cache.FilterUnchangedFiles(files) - if len(files) == 0: - print 'No changes in Torque files detected. Skipping Torque lint check.' - return True - - torquelint = self.GetTorquelintScript() - if torquelint is None: - print('Could not find format-torque.') - sys.exit(1) - - command = [sys.executable, torquelint, '-l'] - - commands = [command + [file] for file in files] - count = multiprocessing.cpu_count() - pool = multiprocessing.Pool(count) - try: - results = pool.map_async(TorqueLintWorker, commands).get() - except KeyboardInterrupt: - print "\nCaught KeyboardInterrupt, terminating workers." - sys.exit(1) - - for i in range(len(files)): - if results[i] > 0: - good_files_cache.RemoveFile(files[i]) + return torque_path, arguments - total_errors = sum(results) - print "Total Torque files requiring formatting: %d" % total_errors - good_files_cache.Save() - return total_errors == 0 + return None, arguments COPYRIGHT_HEADER_PATTERN = re.compile( r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.') @@ -639,6 +665,7 @@ def PyTests(workspace): print 'Running ' + script result &= subprocess.call( [sys.executable, script], stdout=subprocess.PIPE) == 0 + return result @@ -646,6 +673,9 @@ def GetOptions(): result = optparse.OptionParser() result.add_option('--no-lint', help="Do not run cpplint", default=False, action="store_true") + result.add_option('--no-linter-cache', help="Do not cache linter results", default=False, + action="store_true") + return result @@ -656,11 +686,13 @@ def Main(): success = True print "Running checkdeps..." success &= CheckDeps(workspace) + use_linter_cache = not options.no_linter_cache if not options.no_lint: print "Running C++ lint check..." - success &= CppLintProcessor().RunOnPath(workspace) + success &= CppLintProcessor(use_cache=use_linter_cache).RunOnPath(workspace) + print "Running Torque formatting check..." - success &= TorqueFormatProcessor().RunOnPath(workspace) + success &= TorqueFormatProcessor(use_cache=use_linter_cache).RunOnPath(workspace) print "Running copyright header, trailing whitespaces and " \ "two empty lines between declarations check..." success &= SourceProcessor().RunOnPath(workspace) -- cgit v1.2.3