diff options
Diffstat (limited to 'deps/v8/src/code-stub-assembler.cc')
| -rw-r--r-- | deps/v8/src/code-stub-assembler.cc | 1011 |
1 files changed, 651 insertions, 360 deletions
diff --git a/deps/v8/src/code-stub-assembler.cc b/deps/v8/src/code-stub-assembler.cc index d39d841fbd..2527e89a25 100644 --- a/deps/v8/src/code-stub-assembler.cc +++ b/deps/v8/src/code-stub-assembler.cc @@ -10,6 +10,7 @@ #include "src/objects/api-callbacks.h" #include "src/objects/descriptor-array.h" #include "src/objects/ordered-hash-table-inl.h" +#include "src/wasm/wasm-objects.h" namespace v8 { namespace internal { @@ -159,6 +160,14 @@ void CodeStubAssembler::Check(const NodeGenerator& condition_body, extra_node4_name, extra_node5, extra_node5_name); } +void CodeStubAssembler::FastCheck(TNode<BoolT> condition) { + Label ok(this); + GotoIf(condition, &ok); + DebugBreak(); + Goto(&ok); + BIND(&ok); +} + Node* CodeStubAssembler::SelectImpl(TNode<BoolT> condition, const NodeGenerator& true_body, const NodeGenerator& false_body, @@ -1056,12 +1065,10 @@ TNode<BoolT> CodeStubAssembler::IsFastJSArray(SloppyTNode<Object> object, } TNode<BoolT> CodeStubAssembler::IsFastJSArrayWithNoCustomIteration( - TNode<Object> object, TNode<Context> context, - TNode<Context> native_context) { + TNode<Object> object, TNode<Context> context) { Label if_false(this, Label::kDeferred), if_fast(this), exit(this); TVARIABLE(BoolT, var_result); - GotoIfForceSlowPath(&if_false); - BranchIfFastJSArray(object, context, &if_fast, &if_false); + BranchIfFastJSArray(object, context, &if_fast, &if_false, true); BIND(&if_fast); { // Check that the Array.prototype hasn't been modified in a way that would @@ -1083,7 +1090,8 @@ TNode<BoolT> CodeStubAssembler::IsFastJSArrayWithNoCustomIteration( } void CodeStubAssembler::BranchIfFastJSArray(Node* object, Node* context, - Label* if_true, Label* if_false) { + Label* if_true, Label* if_false, + bool iteration_only) { GotoIfForceSlowPath(if_false); // Bailout if receiver is a Smi. @@ -1099,6 +1107,11 @@ void CodeStubAssembler::BranchIfFastJSArray(Node* object, Node* context, // Verify that our prototype is the initial array prototype. GotoIfNot(IsPrototypeInitialArrayPrototype(context, map), if_false); + if (iteration_only) { + // If we are only iterating over the array, there is no need to check + // the NoElements protector if the array is not holey. + GotoIfNot(IsHoleyFastElementsKind(elements_kind), if_true); + } Branch(IsNoElementsProtectorCellInvalid(), if_false, if_true); } @@ -1514,7 +1527,7 @@ TNode<BoolT> CodeStubAssembler::TaggedDoesntHaveInstanceType( TNode<HeapObject> CodeStubAssembler::LoadFastProperties( SloppyTNode<JSObject> object) { - CSA_SLOW_ASSERT(this, Word32Not(IsDictionaryMap(LoadMap(object)))); + CSA_SLOW_ASSERT(this, Word32BinaryNot(IsDictionaryMap(LoadMap(object)))); TNode<Object> properties = LoadObjectField(object, JSObject::kPropertiesOrHashOffset); return Select<HeapObject>(TaggedIsSmi(properties), @@ -1750,8 +1763,8 @@ TNode<IntPtrT> CodeStubAssembler::LoadJSReceiverIdentityHash( BIND(&if_property_dictionary); { - var_hash = SmiUntag(CAST( - LoadFixedArrayElement(properties, NameDictionary::kObjectHashIndex))); + var_hash = SmiUntag(CAST(LoadFixedArrayElement( + CAST(properties), NameDictionary::kObjectHashIndex))); Goto(&done); } @@ -1806,23 +1819,6 @@ Node* CodeStubAssembler::LoadJSValueValue(Node* object) { return LoadObjectField(object, JSValue::kValueOffset); } -TNode<Object> CodeStubAssembler::LoadWeakCellValueUnchecked( - SloppyTNode<HeapObject> weak_cell) { - CSA_ASSERT(this, IsStrongHeapObject(weak_cell)); - // TODO(ishell): fix callers. - return LoadObjectField(weak_cell, WeakCell::kValueOffset); -} - -TNode<Object> CodeStubAssembler::LoadWeakCellValue( - SloppyTNode<WeakCell> weak_cell, Label* if_cleared) { - CSA_ASSERT(this, IsWeakCell(weak_cell)); - TNode<Object> value = LoadWeakCellValueUnchecked(weak_cell); - if (if_cleared != nullptr) { - GotoIf(WordEqual(value, IntPtrConstant(0)), if_cleared); - } - return value; -} - void CodeStubAssembler::DispatchMaybeObject(TNode<MaybeObject> maybe_object, Label* if_smi, Label* if_cleared, Label* if_weak, Label* if_strong, @@ -1962,15 +1958,42 @@ TNode<MaybeObject> CodeStubAssembler::LoadArrayElement( Load(MachineType::AnyTagged(), array, offset, needs_poisoning)); } +void CodeStubAssembler::FixedArrayBoundsCheck(TNode<FixedArrayBase> array, + Node* index, + int additional_offset, + ParameterMode parameter_mode) { + if (!FLAG_fixed_array_bounds_checks) return; + DCHECK_EQ(0, additional_offset % kPointerSize); + if (parameter_mode == ParameterMode::SMI_PARAMETERS) { + TNode<Smi> effective_index; + Smi* constant_index; + bool index_is_constant = ToSmiConstant(index, constant_index); + if (index_is_constant) { + effective_index = SmiConstant(Smi::ToInt(constant_index) + + additional_offset / kPointerSize); + } else if (additional_offset != 0) { + effective_index = + SmiAdd(CAST(index), SmiConstant(additional_offset / kPointerSize)); + } else { + effective_index = CAST(index); + } + CSA_CHECK(this, SmiBelow(effective_index, LoadFixedArrayBaseLength(array))); + } else { + // IntPtrAdd does constant-folding automatically. + TNode<IntPtrT> effective_index = + IntPtrAdd(UncheckedCast<IntPtrT>(index), + IntPtrConstant(additional_offset / kPointerSize)); + CSA_CHECK(this, UintPtrLessThan(effective_index, + LoadAndUntagFixedArrayBaseLength(array))); + } +} + TNode<Object> CodeStubAssembler::LoadFixedArrayElement( - SloppyTNode<HeapObject> object, Node* index_node, int additional_offset, + TNode<FixedArray> object, Node* index_node, int additional_offset, ParameterMode parameter_mode, LoadSensitivity needs_poisoning) { - // This function is currently used for non-FixedArrays (e.g., PropertyArrays) - // and thus the reasonable assert IsFixedArraySubclass(object) is - // not always true. TODO(marja): Fix. - CSA_SLOW_ASSERT( - this, Word32Or(IsFixedArraySubclass(object), IsPropertyArray(object))); + CSA_ASSERT(this, IsFixedArraySubclass(object)); CSA_ASSERT(this, IsNotWeakFixedArraySubclass(object)); + FixedArrayBoundsCheck(object, index_node, additional_offset, parameter_mode); TNode<MaybeObject> element = LoadArrayElement(object, FixedArray::kHeaderSize, index_node, additional_offset, parameter_mode, needs_poisoning); @@ -1989,6 +2012,13 @@ TNode<Object> CodeStubAssembler::LoadPropertyArrayElement( needs_poisoning)); } +TNode<IntPtrT> CodeStubAssembler::LoadPropertyArrayLength( + TNode<PropertyArray> object) { + TNode<IntPtrT> value = + LoadAndUntagObjectField(object, PropertyArray::kLengthAndHashOffset); + return Signed(DecodeWord<PropertyArray::LengthField>(value)); +} + TNode<RawPtrT> CodeStubAssembler::LoadFixedTypedArrayBackingStore( TNode<FixedTypedArrayBase> typed_array) { // Backing store = external_pointer + base_pointer. @@ -2198,6 +2228,47 @@ Node* CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( } } +TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( + TNode<WordT> data_pointer, TNode<Smi> index, TNode<Int32T> elements_kind) { + TVARIABLE(Numeric, var_result); + Label done(this), if_unknown_type(this, Label::kDeferred); + int32_t elements_kinds[] = { +#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) TYPE##_ELEMENTS, + TYPED_ARRAYS(TYPED_ARRAY_CASE) +#undef TYPED_ARRAY_CASE + }; + +#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) Label if_##type##array(this); + TYPED_ARRAYS(TYPED_ARRAY_CASE) +#undef TYPED_ARRAY_CASE + + Label* elements_kind_labels[] = { +#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) &if_##type##array, + TYPED_ARRAYS(TYPED_ARRAY_CASE) +#undef TYPED_ARRAY_CASE + }; + STATIC_ASSERT(arraysize(elements_kinds) == arraysize(elements_kind_labels)); + + Switch(elements_kind, &if_unknown_type, elements_kinds, elements_kind_labels, + arraysize(elements_kinds)); + + BIND(&if_unknown_type); + Unreachable(); + +#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) \ + BIND(&if_##type##array); \ + { \ + var_result = CAST(LoadFixedTypedArrayElementAsTagged( \ + data_pointer, index, TYPE##_ELEMENTS, SMI_PARAMETERS)); \ + Goto(&done); \ + } + TYPED_ARRAYS(TYPED_ARRAY_CASE) +#undef TYPED_ARRAY_CASE + + BIND(&done); + return var_result.value(); +} + void CodeStubAssembler::StoreFixedTypedArrayElementFromTagged( TNode<Context> context, TNode<FixedTypedArrayBase> elements, TNode<Object> index_node, TNode<Object> value, ElementsKind elements_kind, @@ -2316,6 +2387,70 @@ TNode<Float64T> CodeStubAssembler::LoadFixedDoubleArrayElement( return LoadDoubleWithHoleCheck(object, offset, if_hole, machine_type); } +TNode<Object> CodeStubAssembler::LoadFixedArrayBaseElementAsTagged( + TNode<FixedArrayBase> elements, TNode<IntPtrT> index, + TNode<Int32T> elements_kind, Label* if_accessor, Label* if_hole) { + TVARIABLE(Object, var_result); + Label done(this), if_packed(this), if_holey(this), if_packed_double(this), + if_holey_double(this), if_dictionary(this, Label::kDeferred); + + int32_t kinds[] = {// Handled by if_packed. + PACKED_SMI_ELEMENTS, PACKED_ELEMENTS, + // Handled by if_holey. + HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS, + // Handled by if_packed_double. + PACKED_DOUBLE_ELEMENTS, + // Handled by if_holey_double. + HOLEY_DOUBLE_ELEMENTS}; + Label* labels[] = {// PACKED_{SMI,}_ELEMENTS + &if_packed, &if_packed, + // HOLEY_{SMI,}_ELEMENTS + &if_holey, &if_holey, + // PACKED_DOUBLE_ELEMENTS + &if_packed_double, + // HOLEY_DOUBLE_ELEMENTS + &if_holey_double}; + Switch(elements_kind, &if_dictionary, kinds, labels, arraysize(kinds)); + + BIND(&if_packed); + { + var_result = LoadFixedArrayElement(CAST(elements), index, 0); + Goto(&done); + } + + BIND(&if_holey); + { + var_result = LoadFixedArrayElement(CAST(elements), index); + Branch(WordEqual(var_result.value(), TheHoleConstant()), if_hole, &done); + } + + BIND(&if_packed_double); + { + var_result = AllocateHeapNumberWithValue(LoadFixedDoubleArrayElement( + CAST(elements), index, MachineType::Float64())); + Goto(&done); + } + + BIND(&if_holey_double); + { + var_result = AllocateHeapNumberWithValue(LoadFixedDoubleArrayElement( + CAST(elements), index, MachineType::Float64(), 0, INTPTR_PARAMETERS, + if_hole)); + Goto(&done); + } + + BIND(&if_dictionary); + { + CSA_ASSERT(this, IsDictionaryElementsKind(elements_kind)); + var_result = BasicLoadNumberDictionaryElement(CAST(elements), index, + if_accessor, if_hole); + Goto(&done); + } + + BIND(&done); + return var_result.value(); +} + TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck( SloppyTNode<Object> base, SloppyTNode<IntPtrT> offset, Label* if_hole, MachineType machine_type) { @@ -2428,6 +2563,49 @@ TNode<Map> CodeStubAssembler::LoadJSArrayElementsMap( LoadContextElement(native_context, Context::ArrayMapIndex(kind))); } +TNode<BoolT> CodeStubAssembler::IsGeneratorFunction( + TNode<JSFunction> function) { + TNode<SharedFunctionInfo> const shared_function_info = + CAST(LoadObjectField(function, JSFunction::kSharedFunctionInfoOffset)); + + TNode<Uint32T> const function_kind = + DecodeWord32<SharedFunctionInfo::FunctionKindBits>(LoadObjectField( + shared_function_info, SharedFunctionInfo::kFlagsOffset, + MachineType::Uint32())); + + return TNode<BoolT>::UncheckedCast(Word32Or( + Word32Or( + Word32Or( + Word32Equal(function_kind, + Int32Constant(FunctionKind::kAsyncGeneratorFunction)), + Word32Equal( + function_kind, + Int32Constant(FunctionKind::kAsyncConciseGeneratorMethod))), + Word32Equal(function_kind, + Int32Constant(FunctionKind::kGeneratorFunction))), + Word32Equal(function_kind, + Int32Constant(FunctionKind::kConciseGeneratorMethod)))); +} + +TNode<BoolT> CodeStubAssembler::HasPrototypeProperty(TNode<JSFunction> function, + TNode<Map> map) { + // (has_prototype_slot() && IsConstructor()) || + // IsGeneratorFunction(shared()->kind()) + uint32_t mask = + Map::HasPrototypeSlotBit::kMask | Map::IsConstructorBit::kMask; + return TNode<BoolT>::UncheckedCast( + Word32Or(IsAllSetWord32(LoadMapBitField(map), mask), + IsGeneratorFunction(function))); +} + +void CodeStubAssembler::GotoIfPrototypeRequiresRuntimeLookup( + TNode<JSFunction> function, TNode<Map> map, Label* runtime) { + // !has_prototype_property() || has_non_instance_prototype() + GotoIfNot(HasPrototypeProperty(function, map), runtime); + GotoIf(IsSetWord32<Map::HasNonInstancePrototypeBit>(LoadMapBitField(map)), + runtime); +} + Node* CodeStubAssembler::LoadJSFunctionPrototype(Node* function, Label* if_bailout) { CSA_ASSERT(this, TaggedIsNotSmi(function)); @@ -2558,11 +2736,9 @@ Node* CodeStubAssembler::StoreElements(TNode<Object> object, return StoreObjectField(object, JSObject::kElementsOffset, elements); } -Node* CodeStubAssembler::StoreFixedArrayElement(Node* object, Node* index_node, - Node* value, - WriteBarrierMode barrier_mode, - int additional_offset, - ParameterMode parameter_mode) { +void CodeStubAssembler::StoreFixedArrayOrPropertyArrayElement( + Node* object, Node* index_node, Node* value, WriteBarrierMode barrier_mode, + int additional_offset, ParameterMode parameter_mode) { CSA_SLOW_ASSERT( this, Word32Or(IsFixedArraySubclass(object), IsPropertyArray(object))); CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, parameter_mode)); @@ -2597,25 +2773,23 @@ Node* CodeStubAssembler::StoreFixedArrayElement(Node* object, Node* index_node, }), FixedArray::kHeaderSize)); if (barrier_mode == SKIP_WRITE_BARRIER) { - return StoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, - value); + StoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, value); } else { - return Store(object, offset, value); + Store(object, offset, value); } } -Node* CodeStubAssembler::StoreFixedDoubleArrayElement( - Node* object, Node* index_node, Node* value, ParameterMode parameter_mode) { +void CodeStubAssembler::StoreFixedDoubleArrayElement( + TNode<FixedDoubleArray> object, Node* index_node, TNode<Float64T> value, + ParameterMode parameter_mode) { CSA_ASSERT(this, IsFixedDoubleArray(object)); CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, parameter_mode)); + FixedArrayBoundsCheck(object, index_node, 0, parameter_mode); Node* offset = ElementOffsetFromIndex(index_node, PACKED_DOUBLE_ELEMENTS, parameter_mode, FixedArray::kHeaderSize - kHeapObjectTag); - CSA_ASSERT(this, IsOffsetInBounds( - offset, LoadAndUntagFixedArrayBaseLength(object), - FixedDoubleArray::kHeaderSize, PACKED_DOUBLE_ELEMENTS)); MachineRepresentation rep = MachineRepresentation::kFloat64; - return StoreNoWriteBarrier(rep, object, offset, value); + StoreNoWriteBarrier(rep, object, offset, value); } Node* CodeStubAssembler::StoreFeedbackVectorSlot(Node* object, @@ -3189,7 +3363,8 @@ TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionaryWithCapacity( TNode<WordT> store_size = IntPtrAdd( TimesPointerSize(length), IntPtrConstant(NameDictionary::kHeaderSize)); - Node* result = AllocateInNewSpace(store_size); + TNode<NameDictionary> result = + UncheckedCast<NameDictionary>(AllocateInNewSpace(store_size)); Comment("Initialize NameDictionary"); // Initialize FixedArray fields. DCHECK(Heap::RootIsImmortalImmovable(Heap::kNameDictionaryMapRootIndex)); @@ -3222,7 +3397,7 @@ TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionaryWithCapacity( TNode<WordT> end_address = IntPtrAdd( result_word, IntPtrSub(store_size, IntPtrConstant(kHeapObjectTag))); StoreFieldsNoWriteBarrier(start_address, end_address, filler); - return CAST(result); + return result; } TNode<NameDictionary> CodeStubAssembler::CopyNameDictionary( @@ -3260,8 +3435,8 @@ Node* CodeStubAssembler::AllocateOrderedHashTable() { TNode<Map> fixed_array_map = CAST(LoadRoot( static_cast<Heap::RootListIndex>(CollectionType::GetMapRootIndex()))); TNode<FixedArray> table = - AllocateFixedArray(elements_kind, length_intptr, - kAllowLargeObjectAllocation, fixed_array_map); + CAST(AllocateFixedArray(elements_kind, length_intptr, + kAllowLargeObjectAllocation, fixed_array_map)); // Initialize the OrderedHashTable fields. const WriteBarrierMode barrier_mode = SKIP_WRITE_BARRIER; @@ -3383,12 +3558,13 @@ void CodeStubAssembler::FindOrderedHashTableEntry( std::function<void(Node*, Label*, Label*)> key_compare, Variable* entry_start_position, Label* entry_found, Label* not_found) { // Get the index of the bucket. - Node* const number_of_buckets = SmiUntag(CAST( - LoadFixedArrayElement(table, CollectionType::kNumberOfBucketsIndex))); + Node* const number_of_buckets = SmiUntag(CAST(LoadFixedArrayElement( + CAST(table), CollectionType::kNumberOfBucketsIndex))); Node* const bucket = WordAnd(hash, IntPtrSub(number_of_buckets, IntPtrConstant(1))); Node* const first_entry = SmiUntag(CAST(LoadFixedArrayElement( - table, bucket, CollectionType::kHashTableStartIndex * kPointerSize))); + CAST(table), bucket, + CollectionType::kHashTableStartIndex * kPointerSize))); // Walk the bucket chain. Node* entry_start; @@ -3407,14 +3583,14 @@ void CodeStubAssembler::FindOrderedHashTableEntry( // Make sure the entry index is within range. CSA_ASSERT( - this, - UintPtrLessThan( - var_entry.value(), - SmiUntag(SmiAdd( - CAST(LoadFixedArrayElement( - table, CollectionType::kNumberOfElementsIndex)), - CAST(LoadFixedArrayElement( - table, CollectionType::kNumberOfDeletedElementsIndex)))))); + this, UintPtrLessThan( + var_entry.value(), + SmiUntag(SmiAdd( + CAST(LoadFixedArrayElement( + CAST(table), CollectionType::kNumberOfElementsIndex)), + CAST(LoadFixedArrayElement( + CAST(table), + CollectionType::kNumberOfDeletedElementsIndex)))))); // Compute the index of the entry relative to kHashTableStartIndex. entry_start = @@ -3424,7 +3600,7 @@ void CodeStubAssembler::FindOrderedHashTableEntry( // Load the key from the entry. Node* const candidate_key = LoadFixedArrayElement( - table, entry_start, + CAST(table), entry_start, CollectionType::kHashTableStartIndex * kPointerSize); key_compare(candidate_key, &if_key_found, &continue_next_entry); @@ -3432,7 +3608,7 @@ void CodeStubAssembler::FindOrderedHashTableEntry( BIND(&continue_next_entry); // Load the index of the next entry in the bucket chain. var_entry.Bind(SmiUntag(CAST(LoadFixedArrayElement( - table, entry_start, + CAST(table), entry_start, (CollectionType::kHashTableStartIndex + CollectionType::kChainOffset) * kPointerSize)))); @@ -3777,24 +3953,26 @@ Node* CodeStubAssembler::ExtractFastJSArray(Node* context, Node* array, Node* CodeStubAssembler::CloneFastJSArray(Node* context, Node* array, ParameterMode mode, Node* allocation_site) { - Node* length = LoadJSArrayLength(array); - Node* elements = LoadElements(array); - Node* original_array_map = LoadMap(array); Node* elements_kind = LoadMapElementsKind(original_array_map); - Node* new_elements = CloneFixedArray(elements); + Node* length = LoadJSArrayLength(array); + Node* new_elements = ExtractFixedArray( + LoadElements(array), IntPtrOrSmiConstant(0, mode), + TaggedToParameter(length, mode), nullptr, + ExtractFixedArrayFlag::kAllFixedArraysDontCopyCOW, mode); // Use the cannonical map for the Array's ElementsKind Node* native_context = LoadNativeContext(context); Node* array_map = LoadJSArrayElementsMap(elements_kind, native_context); + Node* result = AllocateUninitializedJSArrayWithoutElements(array_map, length, allocation_site); StoreObjectField(result, JSObject::kElementsOffset, new_elements); return result; } -TNode<FixedArray> CodeStubAssembler::AllocateFixedArray( +TNode<FixedArrayBase> CodeStubAssembler::AllocateFixedArray( ElementsKind kind, Node* capacity, ParameterMode mode, AllocationFlags flags, SloppyTNode<Map> fixed_array_map) { Comment("AllocateFixedArray"); @@ -3830,7 +4008,7 @@ TNode<FixedArray> CodeStubAssembler::AllocateFixedArray( return UncheckedCast<FixedArray>(array); } -TNode<FixedArray> CodeStubAssembler::ExtractFixedArray( +TNode<FixedArrayBase> CodeStubAssembler::ExtractFixedArray( Node* fixed_array, Node* first, Node* count, Node* capacity, ExtractFixedArrayFlags extract_flags, ParameterMode parameter_mode) { VARIABLE(var_result, MachineRepresentation::kTagged); @@ -3935,11 +4113,11 @@ TNode<FixedArray> CodeStubAssembler::ExtractFixedArray( BIND(&cow); { if (extract_flags & ExtractFixedArrayFlag::kDontCopyCOW) { - GotoIf(WordNotEqual(IntPtrOrSmiConstant(0, parameter_mode), first), - &new_space_check); - - var_result.Bind(fixed_array); - Goto(&done); + Branch(WordNotEqual(IntPtrOrSmiConstant(0, parameter_mode), first), + &new_space_check, [&] { + var_result.Bind(fixed_array); + Goto(&done); + }); } else { var_fixed_array_map.Bind(LoadRoot(Heap::kFixedArrayMapRootIndex)); Goto(&new_space_check); @@ -4056,6 +4234,48 @@ void CodeStubAssembler::FillFixedArrayWithValue( mode); } +void CodeStubAssembler::FillFixedArrayWithSmiZero(TNode<FixedArray> array, + TNode<IntPtrT> length) { + CSA_ASSERT(this, WordEqual(length, LoadAndUntagFixedArrayBaseLength(array))); + + TNode<IntPtrT> byte_length = TimesPointerSize(length); + CSA_ASSERT(this, UintPtrLessThan(length, byte_length)); + + static const int32_t fa_base_data_offset = + FixedArray::kHeaderSize - kHeapObjectTag; + TNode<IntPtrT> backing_store = IntPtrAdd(BitcastTaggedToWord(array), + IntPtrConstant(fa_base_data_offset)); + + // Call out to memset to perform initialization. + TNode<ExternalReference> memset = + ExternalConstant(ExternalReference::libc_memset_function()); + STATIC_ASSERT(kSizetSize == kIntptrSize); + CallCFunction3(MachineType::Pointer(), MachineType::Pointer(), + MachineType::IntPtr(), MachineType::UintPtr(), memset, + backing_store, IntPtrConstant(0), byte_length); +} + +void CodeStubAssembler::FillFixedDoubleArrayWithZero( + TNode<FixedDoubleArray> array, TNode<IntPtrT> length) { + CSA_ASSERT(this, WordEqual(length, LoadAndUntagFixedArrayBaseLength(array))); + + TNode<IntPtrT> byte_length = TimesDoubleSize(length); + CSA_ASSERT(this, UintPtrLessThan(length, byte_length)); + + static const int32_t fa_base_data_offset = + FixedDoubleArray::kHeaderSize - kHeapObjectTag; + TNode<IntPtrT> backing_store = IntPtrAdd(BitcastTaggedToWord(array), + IntPtrConstant(fa_base_data_offset)); + + // Call out to memset to perform initialization. + TNode<ExternalReference> memset = + ExternalConstant(ExternalReference::libc_memset_function()); + STATIC_ASSERT(kSizetSize == kIntptrSize); + CallCFunction3(MachineType::Pointer(), MachineType::Pointer(), + MachineType::IntPtr(), MachineType::UintPtr(), memset, + backing_store, IntPtrConstant(0), byte_length); +} + void CodeStubAssembler::CopyFixedArrayElements( ElementsKind from_kind, Node* from_array, ElementsKind to_kind, Node* to_array, Node* first_element, Node* element_count, Node* capacity, @@ -4203,8 +4423,8 @@ void CodeStubAssembler::CopyFixedArrayElements( Comment("] CopyFixedArrayElements"); } -TNode<FixedArray> CodeStubAssembler::ConvertFixedArrayBaseToFixedArray( - TNode<FixedArrayBase> base, Label* cast_fail) { +TNode<FixedArray> CodeStubAssembler::HeapObjectToFixedArray( + TNode<HeapObject> base, Label* cast_fail) { Label fixed_array(this); TNode<Map> map = LoadMap(base); GotoIf(WordEqual(map, LoadRoot(Heap::kFixedArrayMapRootIndex)), &fixed_array); @@ -4610,55 +4830,61 @@ Node* CodeStubAssembler::TruncateHeapNumberValueToWord32(Node* object) { return TruncateFloat64ToWord32(value); } -TNode<Number> CodeStubAssembler::ChangeFloat64ToTagged( - SloppyTNode<Float64T> value) { +void CodeStubAssembler::TryHeapNumberToSmi(TNode<HeapNumber> number, + TVariable<Smi>& var_result_smi, + Label* if_smi) { + TNode<Float64T> value = LoadHeapNumberValue(number); + TryFloat64ToSmi(value, var_result_smi, if_smi); +} + +void CodeStubAssembler::TryFloat64ToSmi(TNode<Float64T> value, + TVariable<Smi>& var_result_smi, + Label* if_smi) { TNode<Int32T> value32 = RoundFloat64ToInt32(value); TNode<Float64T> value64 = ChangeInt32ToFloat64(value32); - Label if_valueisint32(this), if_valueisheapnumber(this), if_join(this); + Label if_int32(this), if_heap_number(this, Label::kDeferred); - Label if_valueisequal(this), if_valueisnotequal(this); - Branch(Float64Equal(value, value64), &if_valueisequal, &if_valueisnotequal); - BIND(&if_valueisequal); - { - GotoIfNot(Word32Equal(value32, Int32Constant(0)), &if_valueisint32); - Branch(Int32LessThan(UncheckedCast<Int32T>(Float64ExtractHighWord32(value)), - Int32Constant(0)), - &if_valueisheapnumber, &if_valueisint32); - } - BIND(&if_valueisnotequal); - Goto(&if_valueisheapnumber); + GotoIfNot(Float64Equal(value, value64), &if_heap_number); + GotoIfNot(Word32Equal(value32, Int32Constant(0)), &if_int32); + Branch(Int32LessThan(UncheckedCast<Int32T>(Float64ExtractHighWord32(value)), + Int32Constant(0)), + &if_heap_number, &if_int32); TVARIABLE(Number, var_result); - BIND(&if_valueisint32); + BIND(&if_int32); { if (SmiValuesAre32Bits()) { - TNode<Smi> result = SmiTag(ChangeInt32ToIntPtr(value32)); - var_result = result; - Goto(&if_join); + var_result_smi = SmiTag(ChangeInt32ToIntPtr(value32)); } else { DCHECK(SmiValuesAre31Bits()); TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(value32, value32); TNode<BoolT> overflow = Projection<1>(pair); - Label if_overflow(this, Label::kDeferred), if_notoverflow(this); - Branch(overflow, &if_overflow, &if_notoverflow); - BIND(&if_overflow); - Goto(&if_valueisheapnumber); - BIND(&if_notoverflow); - { - TNode<Smi> result = - BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Projection<0>(pair))); - var_result = result; - Goto(&if_join); - } + GotoIf(overflow, &if_heap_number); + var_result_smi = + BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Projection<0>(pair))); } + Goto(if_smi); } - BIND(&if_valueisheapnumber); + BIND(&if_heap_number); +} + +TNode<Number> CodeStubAssembler::ChangeFloat64ToTagged( + SloppyTNode<Float64T> value) { + Label if_smi(this), done(this); + TVARIABLE(Smi, var_smi_result); + TVARIABLE(Number, var_result); + TryFloat64ToSmi(value, var_smi_result, &if_smi); + + var_result = AllocateHeapNumberWithValue(value); + Goto(&done); + + BIND(&if_smi); { - var_result = AllocateHeapNumberWithValue(value); - Goto(&if_join); + var_result = var_smi_result.value(); + Goto(&done); } - BIND(&if_join); + BIND(&done); return var_result.value(); } @@ -4679,17 +4905,16 @@ TNode<Number> CodeStubAssembler::ChangeInt32ToTagged( TNode<Float64T> value64 = ChangeInt32ToFloat64(value); TNode<HeapNumber> result = AllocateHeapNumberWithValue(value64); var_result = result; + Goto(&if_join); } - Goto(&if_join); BIND(&if_notoverflow); { TNode<IntPtrT> almost_tagged_value = ChangeInt32ToIntPtr(Projection<0>(pair)); - TNode<Smi> result; - result = BitcastWordToTaggedSigned(almost_tagged_value); + TNode<Smi> result = BitcastWordToTaggedSigned(almost_tagged_value); var_result = result; + Goto(&if_join); } - Goto(&if_join); BIND(&if_join); return var_result.value(); } @@ -4780,6 +5005,24 @@ TNode<String> CodeStubAssembler::ToThisString(Node* context, Node* value, return CAST(var_value.value()); } +TNode<Uint32T> CodeStubAssembler::ChangeNumberToUint32(TNode<Number> value) { + TVARIABLE(Uint32T, var_result); + Label if_smi(this), if_heapnumber(this, Label::kDeferred), done(this); + Branch(TaggedIsSmi(value), &if_smi, &if_heapnumber); + BIND(&if_smi); + { + var_result = Unsigned(SmiToInt32(CAST(value))); + Goto(&done); + } + BIND(&if_heapnumber); + { + var_result = ChangeFloat64ToUint32(LoadHeapNumberValue(CAST(value))); + Goto(&done); + } + BIND(&done); + return var_result.value(); +} + TNode<Float64T> CodeStubAssembler::ChangeNumberToFloat64( SloppyTNode<Number> value) { // TODO(tebbi): Remove assert once argument is TNode instead of SloppyTNode. @@ -4804,25 +5047,30 @@ TNode<Float64T> CodeStubAssembler::ChangeNumberToFloat64( TNode<UintPtrT> CodeStubAssembler::ChangeNonnegativeNumberToUintPtr( TNode<Number> value) { TVARIABLE(UintPtrT, result); - Label smi(this), done(this, &result); - GotoIf(TaggedIsSmi(value), &smi); - - TNode<HeapNumber> value_hn = CAST(value); - result = ChangeFloat64ToUintPtr(LoadHeapNumberValue(value_hn)); - Goto(&done); - - BIND(&smi); - TNode<Smi> value_smi = CAST(value); - CSA_SLOW_ASSERT(this, SmiLessThan(SmiConstant(-1), value_smi)); - result = UncheckedCast<UintPtrT>(SmiToIntPtr(value_smi)); - Goto(&done); + Label done(this, &result); + Branch(TaggedIsSmi(value), + [&] { + TNode<Smi> value_smi = CAST(value); + CSA_SLOW_ASSERT(this, SmiLessThan(SmiConstant(-1), value_smi)); + result = UncheckedCast<UintPtrT>(SmiToIntPtr(value_smi)); + Goto(&done); + }, + [&] { + TNode<HeapNumber> value_hn = CAST(value); + result = ChangeFloat64ToUintPtr(LoadHeapNumberValue(value_hn)); + Goto(&done); + }); BIND(&done); return result.value(); } -SloppyTNode<WordT> CodeStubAssembler::TimesPointerSize(Node* value) { - return WordShl(value, IntPtrConstant(kPointerSizeLog2)); +TNode<WordT> CodeStubAssembler::TimesPointerSize(SloppyTNode<WordT> value) { + return WordShl(value, kPointerSizeLog2); +} + +TNode<WordT> CodeStubAssembler::TimesDoubleSize(SloppyTNode<WordT> value) { + return WordShl(value, kDoubleSizeLog2); } Node* CodeStubAssembler::ToThisValue(Node* context, Node* value, @@ -5374,11 +5622,6 @@ TNode<BoolT> CodeStubAssembler::IsFixedArrayWithKind( } } -TNode<BoolT> CodeStubAssembler::IsWeakCell(SloppyTNode<HeapObject> object) { - CSA_ASSERT(this, IsStrongHeapObject(object)); - return IsWeakCellMap(LoadMap(object)); -} - TNode<BoolT> CodeStubAssembler::IsBoolean(SloppyTNode<HeapObject> object) { return IsBooleanMap(LoadMap(object)); } @@ -5676,7 +5919,7 @@ TNode<BoolT> CodeStubAssembler::IsHeapNumberUint32(TNode<HeapNumber> number) { IsHeapNumberPositive(number), [=] { TNode<Float64T> value = LoadHeapNumberValue(number); - TNode<Uint32T> int_value = ChangeFloat64ToUint32(value); + TNode<Uint32T> int_value = Unsigned(TruncateFloat64ToWord32(value)); return Float64Equal(value, ChangeUint32ToFloat64(int_value)); }, [=] { return Int32FalseConstant(); }); @@ -5758,8 +6001,9 @@ TNode<String> CodeStubAssembler::StringFromSingleCharCode(TNode<Int32T> code) { BIND(&if_codeisonebyte); { // Load the isolate wide single character string cache. - Node* cache = LoadRoot(Heap::kSingleCharacterStringCacheRootIndex); - Node* code_index = ChangeUint32ToWord(code); + TNode<FixedArray> cache = + CAST(LoadRoot(Heap::kSingleCharacterStringCacheRootIndex)); + TNode<IntPtrT> code_index = Signed(ChangeUint32ToWord(code)); // Check if we have an entry for the {code} in the single character string // cache already. @@ -5771,7 +6015,7 @@ TNode<String> CodeStubAssembler::StringFromSingleCharCode(TNode<Int32T> code) { BIND(&if_entryisundefined); { // Allocate a new SeqOneByteString for {code} and store it in the {cache}. - Node* result = AllocateSeqOneByteString(1); + TNode<String> result = AllocateSeqOneByteString(1); StoreNoWriteBarrier( MachineRepresentation::kWord8, result, IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag), code); @@ -6081,11 +6325,11 @@ TNode<String> ToDirectStringAssembler::TryToDirect(Label* if_bailout) { return CAST(var_string_.value()); } -Node* ToDirectStringAssembler::TryToSequential(StringPointerKind ptr_kind, - Label* if_bailout) { +TNode<RawPtrT> ToDirectStringAssembler::TryToSequential( + StringPointerKind ptr_kind, Label* if_bailout) { CHECK(ptr_kind == PTR_TO_DATA || ptr_kind == PTR_TO_STRING); - VARIABLE(var_result, MachineType::PointerRepresentation()); + TVARIABLE(RawPtrT, var_result); Label out(this), if_issequential(this), if_isexternal(this, Label::kDeferred); Branch(is_external(), &if_isexternal, &if_issequential); @@ -6093,12 +6337,12 @@ Node* ToDirectStringAssembler::TryToSequential(StringPointerKind ptr_kind, { STATIC_ASSERT(SeqOneByteString::kHeaderSize == SeqTwoByteString::kHeaderSize); - Node* result = BitcastTaggedToWord(var_string_.value()); + TNode<IntPtrT> result = BitcastTaggedToWord(var_string_.value()); if (ptr_kind == PTR_TO_DATA) { result = IntPtrAdd(result, IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag)); } - var_result.Bind(result); + var_result = ReinterpretCast<RawPtrT>(result); Goto(&out); } @@ -6107,14 +6351,14 @@ Node* ToDirectStringAssembler::TryToSequential(StringPointerKind ptr_kind, GotoIf(IsShortExternalStringInstanceType(var_instance_type_.value()), if_bailout); - Node* const string = var_string_.value(); - Node* result = LoadObjectField(string, ExternalString::kResourceDataOffset, - MachineType::Pointer()); + TNode<String> string = CAST(var_string_.value()); + TNode<IntPtrT> result = + LoadObjectField<IntPtrT>(string, ExternalString::kResourceDataOffset); if (ptr_kind == PTR_TO_STRING) { result = IntPtrSub(result, IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag)); } - var_result.Bind(result); + var_result = ReinterpretCast<RawPtrT>(result); Goto(&out); } @@ -6398,7 +6642,9 @@ TNode<Number> CodeStubAssembler::StringToNumber(TNode<String> input) { TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { TVARIABLE(String, result); - Label runtime(this, Label::kDeferred), smi(this), done(this, &result); + TVARIABLE(Smi, smi_input); + Label runtime(this, Label::kDeferred), if_smi(this), if_heap_number(this), + done(this, &result); // Load the number string cache. Node* number_string_cache = LoadRoot(Heap::kNumberStringCacheRootIndex); @@ -6411,63 +6657,67 @@ TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { TNode<IntPtrT> one = IntPtrConstant(1); mask = IntPtrSub(mask, one); - GotoIf(TaggedIsSmi(input), &smi); - - TNode<HeapNumber> heap_number_input = CAST(input); - - // Make a hash from the two 32-bit values of the double. - TNode<Int32T> low = - LoadObjectField<Int32T>(heap_number_input, HeapNumber::kValueOffset); - TNode<Int32T> high = LoadObjectField<Int32T>( - heap_number_input, HeapNumber::kValueOffset + kIntSize); - TNode<Word32T> hash = Word32Xor(low, high); - TNode<WordT> word_hash = WordShl(ChangeInt32ToIntPtr(hash), one); - TNode<WordT> index = - WordAnd(word_hash, WordSar(mask, SmiShiftBitsConstant())); - - // Cache entry's key must be a heap number - Node* number_key = LoadFixedArrayElement(number_string_cache, index); - GotoIf(TaggedIsSmi(number_key), &runtime); - GotoIfNot(IsHeapNumber(number_key), &runtime); - - // Cache entry's key must match the heap number value we're looking for. - Node* low_compare = LoadObjectField(number_key, HeapNumber::kValueOffset, - MachineType::Int32()); - Node* high_compare = LoadObjectField( - number_key, HeapNumber::kValueOffset + kIntSize, MachineType::Int32()); - GotoIfNot(Word32Equal(low, low_compare), &runtime); - GotoIfNot(Word32Equal(high, high_compare), &runtime); - - // Heap number match, return value from cache entry. - IncrementCounter(isolate()->counters()->number_to_string_native(), 1); - result = - CAST(LoadFixedArrayElement(number_string_cache, index, kPointerSize)); - Goto(&done); - - BIND(&runtime); - { - // No cache entry, go to the runtime. - result = CAST(CallRuntime(Runtime::kNumberToStringSkipCache, - NoContextConstant(), input)); + GotoIfNot(TaggedIsSmi(input), &if_heap_number); + smi_input = CAST(input); + Goto(&if_smi); + + BIND(&if_heap_number); + { + TNode<HeapNumber> heap_number_input = CAST(input); + // Try normalizing the HeapNumber. + TryHeapNumberToSmi(heap_number_input, smi_input, &if_smi); + + // Make a hash from the two 32-bit values of the double. + TNode<Int32T> low = + LoadObjectField<Int32T>(heap_number_input, HeapNumber::kValueOffset); + TNode<Int32T> high = LoadObjectField<Int32T>( + heap_number_input, HeapNumber::kValueOffset + kIntSize); + TNode<Word32T> hash = Word32Xor(low, high); + TNode<WordT> word_hash = WordShl(ChangeInt32ToIntPtr(hash), one); + TNode<WordT> index = + WordAnd(word_hash, WordSar(mask, SmiShiftBitsConstant())); + + // Cache entry's key must be a heap number + Node* number_key = LoadFixedArrayElement(CAST(number_string_cache), index); + GotoIf(TaggedIsSmi(number_key), &runtime); + GotoIfNot(IsHeapNumber(number_key), &runtime); + + // Cache entry's key must match the heap number value we're looking for. + Node* low_compare = LoadObjectField(number_key, HeapNumber::kValueOffset, + MachineType::Int32()); + Node* high_compare = LoadObjectField( + number_key, HeapNumber::kValueOffset + kIntSize, MachineType::Int32()); + GotoIfNot(Word32Equal(low, low_compare), &runtime); + GotoIfNot(Word32Equal(high, high_compare), &runtime); + + // Heap number match, return value from cache entry. + result = CAST( + LoadFixedArrayElement(CAST(number_string_cache), index, kPointerSize)); + Goto(&done); } - Goto(&done); - BIND(&smi); + BIND(&if_smi); { // Load the smi key, make sure it matches the smi we're looking for. Node* smi_index = BitcastWordToTagged( - WordAnd(WordShl(BitcastTaggedToWord(input), one), mask)); - Node* smi_key = LoadFixedArrayElement(number_string_cache, smi_index, 0, - SMI_PARAMETERS); - GotoIf(WordNotEqual(smi_key, input), &runtime); + WordAnd(WordShl(BitcastTaggedToWord(smi_input.value()), one), mask)); + Node* smi_key = LoadFixedArrayElement(CAST(number_string_cache), smi_index, + 0, SMI_PARAMETERS); + GotoIf(WordNotEqual(smi_key, smi_input.value()), &runtime); // Smi match, return value from cache entry. - IncrementCounter(isolate()->counters()->number_to_string_native(), 1); - result = CAST(LoadFixedArrayElement(number_string_cache, smi_index, + result = CAST(LoadFixedArrayElement(CAST(number_string_cache), smi_index, kPointerSize, SMI_PARAMETERS)); Goto(&done); } + BIND(&runtime); + { + // No cache entry, go to the runtime. + result = + CAST(CallRuntime(Runtime::kNumberToString, NoContextConstant(), input)); + Goto(&done); + } BIND(&done); return result.value(); } @@ -6961,6 +7211,30 @@ TNode<JSReceiver> CodeStubAssembler::ToObject(SloppyTNode<Context> context, return CAST(CallBuiltin(Builtins::kToObject, context, input)); } +TNode<JSReceiver> CodeStubAssembler::ToObject_Inline(TNode<Context> context, + TNode<Object> input) { + TVARIABLE(JSReceiver, result); + Label if_isreceiver(this), if_isnotreceiver(this, Label::kDeferred); + Label done(this); + + BranchIfJSReceiver(input, &if_isreceiver, &if_isnotreceiver); + + BIND(&if_isreceiver); + { + result = CAST(input); + Goto(&done); + } + + BIND(&if_isnotreceiver); + { + result = ToObject(context, input); + Goto(&done); + } + + BIND(&done); + return result.value(); +} + TNode<Smi> CodeStubAssembler::ToSmiIndex(TNode<Object> input, TNode<Context> context, Label* range_error) { @@ -8159,7 +8433,7 @@ void CodeStubAssembler::LoadPropertyFromGlobalDictionary(Node* dictionary, Comment("[ LoadPropertyFromGlobalDictionary"); CSA_ASSERT(this, IsGlobalDictionary(dictionary)); - Node* property_cell = LoadFixedArrayElement(dictionary, name_index); + Node* property_cell = LoadFixedArrayElement(CAST(dictionary), name_index); CSA_ASSERT(this, IsPropertyCell(property_cell)); Node* value = LoadObjectField(property_cell, PropertyCell::kValueOffset); @@ -8248,15 +8522,8 @@ TNode<Object> CodeStubAssembler::CallGetterIfAccessor( LoadObjectField(accessor_info, AccessorInfo::kNameOffset)), if_bailout); - // if (!(has_prototype_slot() && !has_non_instance_prototype())) use - // generic property loading mechanism. - GotoIfNot( - Word32Equal( - Word32And(LoadMapBitField(receiver_map), - Int32Constant(Map::HasPrototypeSlotBit::kMask | - Map::HasNonInstancePrototypeBit::kMask)), - Int32Constant(Map::HasPrototypeSlotBit::kMask)), - if_bailout); + GotoIfPrototypeRequiresRuntimeLookup(CAST(receiver), CAST(receiver_map), + if_bailout); var_value.Bind(LoadJSFunctionPrototype(receiver, if_bailout)); Goto(&done); } @@ -8419,25 +8686,26 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, BIND(&if_isobjectorsmi); { - Node* elements = LoadElements(object); - Node* length = LoadAndUntagFixedArrayBaseLength(elements); + TNode<FixedArray> elements = CAST(LoadElements(object)); + TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(elements); GotoIfNot(UintPtrLessThan(intptr_index, length), &if_oob); - Node* element = LoadFixedArrayElement(elements, intptr_index); - Node* the_hole = TheHoleConstant(); + TNode<Object> element = LoadFixedArrayElement(elements, intptr_index); + TNode<Oddball> the_hole = TheHoleConstant(); Branch(WordEqual(element, the_hole), if_not_found, if_found); } BIND(&if_isdouble); { - Node* elements = LoadElements(object); - Node* length = LoadAndUntagFixedArrayBaseLength(elements); + TNode<FixedArrayBase> elements = LoadElements(object); + TNode<IntPtrT> length = LoadAndUntagFixedArrayBaseLength(elements); GotoIfNot(UintPtrLessThan(intptr_index, length), &if_oob); // Check if the element is a double hole, but don't load it. - LoadFixedDoubleArrayElement(elements, intptr_index, MachineType::None(), 0, - INTPTR_PARAMETERS, if_not_found); + LoadFixedDoubleArrayElement(CAST(elements), intptr_index, + MachineType::None(), 0, INTPTR_PARAMETERS, + if_not_found); Goto(if_found); } BIND(&if_isdictionary); @@ -8709,15 +8977,8 @@ Node* CodeStubAssembler::OrdinaryHasInstance(Node* context, Node* callable, GotoIfNot(InstanceTypeEqual(callable_instance_type, JS_FUNCTION_TYPE), &return_runtime); - // Goto runtime if {callable} is not a constructor or has - // a non-instance "prototype". - Node* callable_bitfield = LoadMapBitField(callable_map); - GotoIfNot(Word32Equal( - Word32And(callable_bitfield, - Int32Constant(Map::HasNonInstancePrototypeBit::kMask | - Map::IsConstructorBit::kMask)), - Int32Constant(Map::IsConstructorBit::kMask)), - &return_runtime); + GotoIfPrototypeRequiresRuntimeLookup(CAST(callable), CAST(callable_map), + &return_runtime); // Get the "prototype" (or initial map) of the {callable}. Node* callable_prototype = @@ -8963,8 +9224,8 @@ Node* CodeStubAssembler::EmitKeyedSloppyArguments(Node* receiver, Node* key, key = SmiUntag(key); GotoIf(IntPtrLessThan(key, IntPtrConstant(0)), bailout); - Node* elements = LoadElements(receiver); - Node* elements_length = LoadAndUntagFixedArrayBaseLength(elements); + TNode<FixedArray> elements = CAST(LoadElements(receiver)); + TNode<IntPtrT> elements_length = LoadAndUntagFixedArrayBaseLength(elements); VARIABLE(var_result, MachineRepresentation::kTagged); if (!is_load) { @@ -8976,37 +9237,38 @@ Node* CodeStubAssembler::EmitKeyedSloppyArguments(Node* receiver, Node* key, GotoIf(UintPtrGreaterThanOrEqual(key, adjusted_length), &if_unmapped); - Node* mapped_index = + TNode<Object> mapped_index = LoadFixedArrayElement(elements, IntPtrAdd(key, intptr_two)); Branch(WordEqual(mapped_index, TheHoleConstant()), &if_unmapped, &if_mapped); BIND(&if_mapped); { - CSA_ASSERT(this, TaggedIsSmi(mapped_index)); - mapped_index = SmiUntag(mapped_index); - Node* the_context = LoadFixedArrayElement(elements, 0); + TNode<IntPtrT> mapped_index_intptr = SmiUntag(CAST(mapped_index)); + TNode<Context> the_context = CAST(LoadFixedArrayElement(elements, 0)); // Assert that we can use LoadFixedArrayElement/StoreFixedArrayElement // methods for accessing Context. STATIC_ASSERT(Context::kHeaderSize == FixedArray::kHeaderSize); DCHECK_EQ(Context::SlotOffset(0) + kHeapObjectTag, FixedArray::OffsetOfElementAt(0)); if (is_load) { - Node* result = LoadFixedArrayElement(the_context, mapped_index); + Node* result = LoadFixedArrayElement(the_context, mapped_index_intptr); CSA_ASSERT(this, WordNotEqual(result, TheHoleConstant())); var_result.Bind(result); } else { - StoreFixedArrayElement(the_context, mapped_index, value); + StoreFixedArrayElement(the_context, mapped_index_intptr, value); } Goto(&end); } BIND(&if_unmapped); { - Node* backing_store = LoadFixedArrayElement(elements, 1); - GotoIf(WordNotEqual(LoadMap(backing_store), FixedArrayMapConstant()), + TNode<HeapObject> backing_store_ho = + CAST(LoadFixedArrayElement(elements, 1)); + GotoIf(WordNotEqual(LoadMap(backing_store_ho), FixedArrayMapConstant()), bailout); + TNode<FixedArray> backing_store = CAST(backing_store_ho); - Node* backing_store_length = + TNode<IntPtrT> backing_store_length = LoadAndUntagFixedArrayBaseLength(backing_store); GotoIf(UintPtrGreaterThanOrEqual(key, backing_store_length), bailout); @@ -9078,12 +9340,12 @@ void CodeStubAssembler::StoreElement(Node* elements, ElementsKind kind, return; } else if (IsDoubleElementsKind(kind)) { // Make sure we do not store signalling NaNs into double arrays. - value = Float64SilenceNaN(value); - StoreFixedDoubleArrayElement(elements, index, value, mode); + TNode<Float64T> value_silenced = Float64SilenceNaN(value); + StoreFixedDoubleArrayElement(CAST(elements), index, value_silenced, mode); } else { WriteBarrierMode barrier_mode = IsSmiElementsKind(kind) ? SKIP_WRITE_BARRIER : UPDATE_WRITE_BARRIER; - StoreFixedArrayElement(elements, index, value, barrier_mode, 0, mode); + StoreFixedArrayElement(CAST(elements), index, value, barrier_mode, 0, mode); } } @@ -9391,12 +9653,13 @@ Node* CodeStubAssembler::CheckForCapacityGrow( Node* object, Node* elements, ElementsKind kind, KeyedAccessStoreMode store_mode, Node* length, Node* key, ParameterMode mode, bool is_js_array, Label* bailout) { + DCHECK(IsFastElementsKind(kind)); VARIABLE(checked_elements, MachineRepresentation::kTagged); Label grow_case(this), no_grow_case(this), done(this), grow_bailout(this, Label::kDeferred); Node* condition; - if (IsHoleyOrDictionaryElementsKind(kind)) { + if (IsHoleyElementsKind(kind)) { condition = UintPtrGreaterThanOrEqual(key, length); } else { // We don't support growing here unless the value is being appended. @@ -9572,7 +9835,7 @@ void CodeStubAssembler::TrapAllocationMemento(Node* object, } TNode<IntPtrT> CodeStubAssembler::PageFromAddress(TNode<IntPtrT> address) { - return WordAnd(address, IntPtrConstant(~Page::kPageAlignmentMask)); + return WordAnd(address, IntPtrConstant(~kPageAlignmentMask)); } TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector( @@ -9603,7 +9866,7 @@ TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector( // Store an empty fixed array for the code dependency. StoreObjectFieldRoot(site, AllocationSite::kDependentCodeOffset, - Heap::kEmptyFixedArrayRootIndex); + Heap::kEmptyWeakFixedArrayRootIndex); // Link the object to the allocation site list TNode<ExternalReference> site_list = ExternalConstant( @@ -9624,10 +9887,13 @@ TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector( } TNode<MaybeObject> CodeStubAssembler::StoreWeakReferenceInFeedbackVector( - SloppyTNode<FeedbackVector> feedback_vector, SloppyTNode<IntPtrT> slot, - TNode<HeapObject> value) { + SloppyTNode<FeedbackVector> feedback_vector, Node* slot, + SloppyTNode<HeapObject> value, int additional_offset, + ParameterMode parameter_mode) { TNode<MaybeObject> weak_value = MakeWeak(value); - StoreFeedbackVectorSlot(feedback_vector, slot, weak_value); + StoreFeedbackVectorSlot(feedback_vector, slot, weak_value, + UPDATE_WRITE_BARRIER, additional_offset, + parameter_mode); return weak_value; } @@ -9681,7 +9947,15 @@ Node* CodeStubAssembler::BuildFastLoop( // to force the loop header check at the end of the loop and branch forward to // it from the pre-header). The extra branch is slower in the case that the // loop actually iterates. - Branch(WordEqual(var.value(), end_index), &after_loop, &loop); + Node* first_check = WordEqual(var.value(), end_index); + int32_t first_check_val; + if (ToInt32Constant(first_check, first_check_val)) { + if (first_check_val) return var.value(); + Goto(&loop); + } else { + Branch(first_check, &after_loop, &loop); + } + BIND(&loop); { if (advance_mode == IndexAdvanceMode::kPre) { @@ -9785,62 +10059,59 @@ void CodeStubAssembler::BranchIfNumberRelationalComparison( TVARIABLE(Float64T, var_left_float); TVARIABLE(Float64T, var_right_float); - Label if_left_smi(this), if_left_not_smi(this); - Branch(TaggedIsSmi(left), &if_left_smi, &if_left_not_smi); - - BIND(&if_left_smi); - { - TNode<Smi> smi_left = CAST(left); - - Label if_right_not_smi(this); - GotoIfNot(TaggedIsSmi(right), &if_right_not_smi); - { - TNode<Smi> smi_right = CAST(right); - - // Both {left} and {right} are Smi, so just perform a fast Smi comparison. - switch (op) { - case Operation::kLessThan: - BranchIfSmiLessThan(smi_left, smi_right, if_true, if_false); - break; - case Operation::kLessThanOrEqual: - BranchIfSmiLessThanOrEqual(smi_left, smi_right, if_true, if_false); - break; - case Operation::kGreaterThan: - BranchIfSmiLessThan(smi_right, smi_left, if_true, if_false); - break; - case Operation::kGreaterThanOrEqual: - BranchIfSmiLessThanOrEqual(smi_right, smi_left, if_true, if_false); - break; - default: - UNREACHABLE(); - } - } - BIND(&if_right_not_smi); - { - CSA_ASSERT(this, IsHeapNumber(right)); - var_left_float = SmiToFloat64(smi_left); - var_right_float = LoadHeapNumberValue(right); - Goto(&do_float_comparison); - } - } - - BIND(&if_left_not_smi); - { - CSA_ASSERT(this, IsHeapNumber(left)); - var_left_float = LoadHeapNumberValue(left); - - Label if_right_not_smi(this); - GotoIfNot(TaggedIsSmi(right), &if_right_not_smi); - var_right_float = SmiToFloat64(right); - Goto(&do_float_comparison); - - BIND(&if_right_not_smi); - { - CSA_ASSERT(this, IsHeapNumber(right)); - var_right_float = LoadHeapNumberValue(right); - Goto(&do_float_comparison); - } - } + Branch(TaggedIsSmi(left), + [&] { + TNode<Smi> smi_left = CAST(left); + + Branch(TaggedIsSmi(right), + [&] { + TNode<Smi> smi_right = CAST(right); + + // Both {left} and {right} are Smi, so just perform a fast + // Smi comparison. + switch (op) { + case Operation::kLessThan: + BranchIfSmiLessThan(smi_left, smi_right, if_true, + if_false); + break; + case Operation::kLessThanOrEqual: + BranchIfSmiLessThanOrEqual(smi_left, smi_right, if_true, + if_false); + break; + case Operation::kGreaterThan: + BranchIfSmiLessThan(smi_right, smi_left, if_true, + if_false); + break; + case Operation::kGreaterThanOrEqual: + BranchIfSmiLessThanOrEqual(smi_right, smi_left, if_true, + if_false); + break; + default: + UNREACHABLE(); + } + }, + [&] { + CSA_ASSERT(this, IsHeapNumber(right)); + var_left_float = SmiToFloat64(smi_left); + var_right_float = LoadHeapNumberValue(right); + Goto(&do_float_comparison); + }); + }, + [&] { + CSA_ASSERT(this, IsHeapNumber(left)); + var_left_float = LoadHeapNumberValue(left); + + Branch(TaggedIsSmi(right), + [&] { + var_right_float = SmiToFloat64(right); + Goto(&do_float_comparison); + }, + [&] { + CSA_ASSERT(this, IsHeapNumber(right)); + var_right_float = LoadHeapNumberValue(right); + Goto(&do_float_comparison); + }); + }); BIND(&do_float_comparison); { @@ -11086,69 +11357,66 @@ void CodeStubAssembler::BranchIfSameValue(Node* lhs, Node* rhs, Label* if_true, { // Since {lhs} is a Smi, the comparison can only yield true // iff the {rhs} is a HeapNumber with the same float64 value. - GotoIf(TaggedIsSmi(rhs), if_false); - GotoIfNot(IsHeapNumber(rhs), if_false); - var_lhs_value.Bind(SmiToFloat64(lhs)); - var_rhs_value.Bind(LoadHeapNumberValue(rhs)); - Goto(&do_fcmp); + Branch(TaggedIsSmi(rhs), if_false, [&] { + GotoIfNot(IsHeapNumber(rhs), if_false); + var_lhs_value.Bind(SmiToFloat64(lhs)); + var_rhs_value.Bind(LoadHeapNumberValue(rhs)); + Goto(&do_fcmp); + }); } BIND(&if_lhsisheapobject); { // Check if the {rhs} is a Smi. - Label if_rhsissmi(this), if_rhsisheapobject(this); - Branch(TaggedIsSmi(rhs), &if_rhsissmi, &if_rhsisheapobject); - - BIND(&if_rhsissmi); - { - // Since {rhs} is a Smi, the comparison can only yield true - // iff the {lhs} is a HeapNumber with the same float64 value. - GotoIfNot(IsHeapNumber(lhs), if_false); - var_lhs_value.Bind(LoadHeapNumberValue(lhs)); - var_rhs_value.Bind(SmiToFloat64(rhs)); - Goto(&do_fcmp); - } - - BIND(&if_rhsisheapobject); - { - // Now this can only yield true if either both {lhs} and {rhs} are - // HeapNumbers with the same value, or both are Strings with the same - // character sequence, or both are BigInts with the same value. - Label if_lhsisheapnumber(this), if_lhsisstring(this), - if_lhsisbigint(this); - Node* const lhs_map = LoadMap(lhs); - GotoIf(IsHeapNumberMap(lhs_map), &if_lhsisheapnumber); - Node* const lhs_instance_type = LoadMapInstanceType(lhs_map); - GotoIf(IsStringInstanceType(lhs_instance_type), &if_lhsisstring); - Branch(IsBigIntInstanceType(lhs_instance_type), &if_lhsisbigint, - if_false); - - BIND(&if_lhsisheapnumber); - { - GotoIfNot(IsHeapNumber(rhs), if_false); - var_lhs_value.Bind(LoadHeapNumberValue(lhs)); - var_rhs_value.Bind(LoadHeapNumberValue(rhs)); - Goto(&do_fcmp); - } - - BIND(&if_lhsisstring); - { - // Now we can only yield true if {rhs} is also a String - // with the same sequence of characters. - GotoIfNot(IsString(rhs), if_false); - Node* const result = - CallBuiltin(Builtins::kStringEqual, NoContextConstant(), lhs, rhs); - Branch(IsTrue(result), if_true, if_false); - } - - BIND(&if_lhsisbigint); - { - GotoIfNot(IsBigInt(rhs), if_false); - Node* const result = CallRuntime(Runtime::kBigIntEqualToBigInt, - NoContextConstant(), lhs, rhs); - Branch(IsTrue(result), if_true, if_false); - } - } + Branch(TaggedIsSmi(rhs), + [&] { + // Since {rhs} is a Smi, the comparison can only yield true + // iff the {lhs} is a HeapNumber with the same float64 value. + GotoIfNot(IsHeapNumber(lhs), if_false); + var_lhs_value.Bind(LoadHeapNumberValue(lhs)); + var_rhs_value.Bind(SmiToFloat64(rhs)); + Goto(&do_fcmp); + }, + [&] { + // Now this can only yield true if either both {lhs} and {rhs} are + // HeapNumbers with the same value, or both are Strings with the + // same character sequence, or both are BigInts with the same + // value. + Label if_lhsisheapnumber(this), if_lhsisstring(this), + if_lhsisbigint(this); + Node* const lhs_map = LoadMap(lhs); + GotoIf(IsHeapNumberMap(lhs_map), &if_lhsisheapnumber); + Node* const lhs_instance_type = LoadMapInstanceType(lhs_map); + GotoIf(IsStringInstanceType(lhs_instance_type), &if_lhsisstring); + Branch(IsBigIntInstanceType(lhs_instance_type), &if_lhsisbigint, + if_false); + + BIND(&if_lhsisheapnumber); + { + GotoIfNot(IsHeapNumber(rhs), if_false); + var_lhs_value.Bind(LoadHeapNumberValue(lhs)); + var_rhs_value.Bind(LoadHeapNumberValue(rhs)); + Goto(&do_fcmp); + } + + BIND(&if_lhsisstring); + { + // Now we can only yield true if {rhs} is also a String + // with the same sequence of characters. + GotoIfNot(IsString(rhs), if_false); + Node* const result = CallBuiltin(Builtins::kStringEqual, + NoContextConstant(), lhs, rhs); + Branch(IsTrue(result), if_true, if_false); + } + + BIND(&if_lhsisbigint); + { + GotoIfNot(IsBigInt(rhs), if_false); + Node* const result = CallRuntime(Runtime::kBigIntEqualToBigInt, + NoContextConstant(), lhs, rhs); + Branch(IsTrue(result), if_true, if_false); + } + }); } BIND(&do_fcmp); @@ -11181,9 +11449,9 @@ void CodeStubAssembler::BranchIfSameValue(Node* lhs, Node* rhs, Label* if_true, } } -TNode<Oddball> CodeStubAssembler::HasProperty(SloppyTNode<HeapObject> object, +TNode<Oddball> CodeStubAssembler::HasProperty(SloppyTNode<Context> context, + SloppyTNode<Object> object, SloppyTNode<Object> key, - SloppyTNode<Context> context, HasPropertyLookupMode mode) { Label call_runtime(this, Label::kDeferred), return_true(this), return_false(this), end(this), if_proxy(this, Label::kDeferred); @@ -11715,8 +11983,8 @@ Node* CodeStubAssembler::AllocateJSIteratorResultForEntry(Node* context, Node* native_context = LoadNativeContext(context); Node* length = SmiConstant(2); int const elements_size = FixedArray::SizeFor(2); - Node* elements = - Allocate(elements_size + JSArray::kSize + JSIteratorResult::kSize); + TNode<FixedArray> elements = UncheckedCast<FixedArray>( + Allocate(elements_size + JSArray::kSize + JSIteratorResult::kSize)); StoreObjectFieldRoot(elements, FixedArray::kMapOffset, Heap::kFixedArrayMapRootIndex); StoreObjectFieldNoWriteBarrier(elements, FixedArray::kLengthOffset, length); @@ -11761,6 +12029,23 @@ Node* CodeStubAssembler::IsDetachedBuffer(Node* buffer) { return IsSetWord32<JSArrayBuffer::WasNeutered>(buffer_bit_field); } +void CodeStubAssembler::ThrowIfArrayBufferIsDetached( + SloppyTNode<Context> context, TNode<JSArrayBuffer> array_buffer, + const char* method_name) { + Label if_detached(this, Label::kDeferred), if_not_detached(this); + Branch(IsDetachedBuffer(array_buffer), &if_detached, &if_not_detached); + BIND(&if_detached); + ThrowTypeError(context, MessageTemplate::kDetachedOperation, method_name); + BIND(&if_not_detached); +} + +void CodeStubAssembler::ThrowIfArrayBufferViewBufferIsDetached( + SloppyTNode<Context> context, TNode<JSArrayBufferView> array_buffer_view, + const char* method_name) { + TNode<JSArrayBuffer> buffer = LoadArrayBufferViewBuffer(array_buffer_view); + ThrowIfArrayBufferIsDetached(context, buffer, method_name); +} + TNode<JSArrayBuffer> CodeStubAssembler::LoadArrayBufferViewBuffer( TNode<JSArrayBufferView> array_buffer_view) { return LoadObjectField<JSArrayBuffer>(array_buffer_view, @@ -11959,6 +12244,13 @@ Node* CodeStubAssembler::IsDebugActive() { return Word32NotEqual(is_debug_active, Int32Constant(0)); } +TNode<BoolT> CodeStubAssembler::IsRuntimeCallStatsEnabled() { + TNode<Word32T> flag_value = UncheckedCast<Word32T>(Load( + MachineType::Int32(), + ExternalConstant(ExternalReference::address_of_runtime_stats_flag()))); + return Word32NotEqual(flag_value, Int32Constant(0)); +} + Node* CodeStubAssembler::IsPromiseHookEnabled() { Node* const promise_hook = Load( MachineType::Pointer(), @@ -12191,9 +12483,8 @@ Node* CodeStubAssembler::CheckEnumCache(Node* receiver, Label* if_empty, { // Avoid runtime-call for empty dictionary receivers. GotoIfNot(IsDictionaryMap(receiver_map), if_runtime); - Node* properties = LoadSlowProperties(receiver); - Node* length = LoadFixedArrayElement( - properties, NameDictionary::kNumberOfElementsIndex); + TNode<NameDictionary> properties = CAST(LoadSlowProperties(receiver)); + TNode<Smi> length = GetNumberOfElements(properties); GotoIfNot(WordEqual(length, SmiConstant(0)), if_runtime); // Check that there are no elements on the {receiver} and its prototype // chain. Given that we do not create an EnumCache for dict-mode objects, |