diff options
Diffstat (limited to 'deps/v8/src/codegen/code-stub-assembler.cc')
| -rw-r--r-- | deps/v8/src/codegen/code-stub-assembler.cc | 2529 |
1 files changed, 1046 insertions, 1483 deletions
diff --git a/deps/v8/src/codegen/code-stub-assembler.cc b/deps/v8/src/codegen/code-stub-assembler.cc index 7dad8cb95e..3051ce3662 100644 --- a/deps/v8/src/codegen/code-stub-assembler.cc +++ b/deps/v8/src/codegen/code-stub-assembler.cc @@ -7,9 +7,11 @@ #include "include/v8-internal.h" #include "src/base/macros.h" #include "src/codegen/code-factory.h" +#include "src/codegen/tnode.h" #include "src/common/globals.h" #include "src/execution/frames-inl.h" #include "src/execution/frames.h" +#include "src/execution/protectors.h" #include "src/heap/heap-inl.h" // For Page/MemoryChunk. TODO(jkummerow): Drop. #include "src/logging/counters.h" #include "src/objects/api-callbacks.h" @@ -17,6 +19,7 @@ #include "src/objects/descriptor-array.h" #include "src/objects/function-kind.h" #include "src/objects/heap-number.h" +#include "src/objects/js-generator.h" #include "src/objects/oddball.h" #include "src/objects/ordered-hash-table-inl.h" #include "src/objects/property-cell.h" @@ -26,10 +29,6 @@ namespace v8 { namespace internal { using compiler::Node; -template <class T> -using TNode = compiler::TNode<T>; -template <class T> -using SloppyTNode = compiler::SloppyTNode<T>; CodeStubAssembler::CodeStubAssembler(compiler::CodeAssemblerState* state) : compiler::CodeAssembler(state), @@ -135,6 +134,148 @@ void CodeStubAssembler::Check(SloppyTNode<Word32T> condition_node, Check(branch, message, file, line, extra_nodes); } +template <> +TNode<Smi> CodeStubAssembler::IntPtrToParameter<Smi>(TNode<IntPtrT> value) { + return SmiTag(value); +} +template <> +TNode<IntPtrT> CodeStubAssembler::IntPtrToParameter<IntPtrT>( + TNode<IntPtrT> value) { + return value; +} + +void CodeStubAssembler::CollectCallableFeedback( + TNode<Object> maybe_target, TNode<Context> context, + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot_id) { + Label extra_checks(this, Label::kDeferred), done(this); + + // Check if we have monomorphic {target} feedback already. + TNode<MaybeObject> feedback = + LoadFeedbackVectorSlot(feedback_vector, slot_id); + Comment("check if monomorphic"); + TNode<BoolT> is_monomorphic = IsWeakReferenceToObject(feedback, maybe_target); + GotoIf(is_monomorphic, &done); + + // Check if it is a megamorphic {target}. + Comment("check if megamorphic"); + TNode<BoolT> is_megamorphic = TaggedEqual( + feedback, HeapConstant(FeedbackVector::MegamorphicSentinel(isolate()))); + Branch(is_megamorphic, &done, &extra_checks); + + BIND(&extra_checks); + { + Label initialize(this), mark_megamorphic(this); + + Comment("check if weak reference"); + TNode<BoolT> is_uninitialized = TaggedEqual( + feedback, + HeapConstant(FeedbackVector::UninitializedSentinel(isolate()))); + GotoIf(is_uninitialized, &initialize); + CSA_ASSERT(this, IsWeakOrCleared(feedback)); + + // If the weak reference is cleared, we have a new chance to become + // monomorphic. + Comment("check if weak reference is cleared"); + Branch(IsCleared(feedback), &initialize, &mark_megamorphic); + + BIND(&initialize); + { + Comment("check if function in same native context"); + GotoIf(TaggedIsSmi(maybe_target), &mark_megamorphic); + TNode<HeapObject> target = CAST(maybe_target); + // Check if the {target} is a JSFunction or JSBoundFunction + // in the current native context. + TVARIABLE(HeapObject, var_current, target); + Label loop(this, &var_current), done_loop(this); + Goto(&loop); + BIND(&loop); + { + Label if_boundfunction(this), if_function(this); + TNode<HeapObject> current = var_current.value(); + TNode<Uint16T> current_instance_type = LoadInstanceType(current); + GotoIf(InstanceTypeEqual(current_instance_type, JS_BOUND_FUNCTION_TYPE), + &if_boundfunction); + Branch(InstanceTypeEqual(current_instance_type, JS_FUNCTION_TYPE), + &if_function, &mark_megamorphic); + + BIND(&if_function); + { + // Check that the JSFunction {current} is in the current native + // context. + TNode<Context> current_context = + CAST(LoadObjectField(current, JSFunction::kContextOffset)); + TNode<NativeContext> current_native_context = + LoadNativeContext(current_context); + Branch( + TaggedEqual(LoadNativeContext(context), current_native_context), + &done_loop, &mark_megamorphic); + } + BIND(&if_boundfunction); + { + // Continue with the [[BoundTargetFunction]] of {target}. + var_current = LoadObjectField<HeapObject>( + current, JSBoundFunction::kBoundTargetFunctionOffset); + Goto(&loop); + } + } + BIND(&done_loop); + StoreWeakReferenceInFeedbackVector(feedback_vector, slot_id, target); + ReportFeedbackUpdate(feedback_vector, slot_id, "Call:Initialize"); + Goto(&done); + } + + BIND(&mark_megamorphic); + { + // MegamorphicSentinel is an immortal immovable object so + // write-barrier is not needed. + Comment("transition to megamorphic"); + DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kmegamorphic_symbol)); + StoreFeedbackVectorSlot( + feedback_vector, slot_id, + HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())), + SKIP_WRITE_BARRIER); + ReportFeedbackUpdate(feedback_vector, slot_id, + "Call:TransitionMegamorphic"); + Goto(&done); + } + } + + BIND(&done); +} + +void CodeStubAssembler::CollectCallFeedback( + TNode<Object> maybe_target, TNode<Context> context, + TNode<HeapObject> maybe_feedback_vector, TNode<UintPtrT> slot_id) { + Label feedback_done(this); + // If feedback_vector is not valid, then nothing to do. + GotoIf(IsUndefined(maybe_feedback_vector), &feedback_done); + + // Increment the call count. + TNode<FeedbackVector> feedback_vector = CAST(maybe_feedback_vector); + IncrementCallCount(feedback_vector, slot_id); + + // Collect the callable {target} feedback. + CollectCallableFeedback(maybe_target, context, feedback_vector, slot_id); + Goto(&feedback_done); + + BIND(&feedback_done); +} + +void CodeStubAssembler::IncrementCallCount( + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot_id) { + Comment("increment call count"); + TNode<Smi> call_count = + CAST(LoadFeedbackVectorSlot(feedback_vector, slot_id, kTaggedSize)); + // The lowest {FeedbackNexus::CallCountField::kShift} bits of the call + // count are used as flags. To increment the call count by 1 we hence + // have to increment by 1 << {FeedbackNexus::CallCountField::kShift}. + TNode<Smi> new_count = SmiAdd( + call_count, SmiConstant(1 << FeedbackNexus::CallCountField::kShift)); + // Count is Smi, so we don't need a write barrier. + StoreFeedbackVectorSlot(feedback_vector, slot_id, new_count, + SKIP_WRITE_BARRIER, kTaggedSize); +} + void CodeStubAssembler::FastCheck(TNode<BoolT> condition) { Label ok(this), not_ok(this, Label::kDeferred); Branch(condition, &ok, ¬_ok); @@ -221,7 +362,7 @@ TNode<Object> CodeStubAssembler::NoContextConstant() { } #define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \ - compiler::TNode<std::remove_pointer<std::remove_reference<decltype( \ + TNode<std::remove_pointer<std::remove_reference<decltype( \ std::declval<Heap>().rootAccessorName())>::type>::type> \ CodeStubAssembler::name##Constant() { \ return UncheckedCast<std::remove_pointer<std::remove_reference<decltype( \ @@ -232,7 +373,7 @@ HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR) #undef HEAP_CONSTANT_ACCESSOR #define HEAP_CONSTANT_ACCESSOR(rootIndexName, rootAccessorName, name) \ - compiler::TNode<std::remove_pointer<std::remove_reference<decltype( \ + TNode<std::remove_pointer<std::remove_reference<decltype( \ std::declval<ReadOnlyRoots>().rootAccessorName())>::type>::type> \ CodeStubAssembler::name##Constant() { \ return UncheckedCast<std::remove_pointer<std::remove_reference<decltype( \ @@ -242,14 +383,12 @@ HEAP_MUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR) HEAP_IMMUTABLE_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_ACCESSOR) #undef HEAP_CONSTANT_ACCESSOR -#define HEAP_CONSTANT_TEST(rootIndexName, rootAccessorName, name) \ - compiler::TNode<BoolT> CodeStubAssembler::Is##name( \ - SloppyTNode<Object> value) { \ - return TaggedEqual(value, name##Constant()); \ - } \ - compiler::TNode<BoolT> CodeStubAssembler::IsNot##name( \ - SloppyTNode<Object> value) { \ - return TaggedNotEqual(value, name##Constant()); \ +#define HEAP_CONSTANT_TEST(rootIndexName, rootAccessorName, name) \ + TNode<BoolT> CodeStubAssembler::Is##name(SloppyTNode<Object> value) { \ + return TaggedEqual(value, name##Constant()); \ + } \ + TNode<BoolT> CodeStubAssembler::IsNot##name(SloppyTNode<Object> value) { \ + return TaggedNotEqual(value, name##Constant()); \ } HEAP_IMMOVABLE_OBJECT_LIST(HEAP_CONSTANT_TEST) #undef HEAP_CONSTANT_TEST @@ -264,6 +403,21 @@ TNode<BInt> CodeStubAssembler::BIntConstant(int value) { #endif } +template <> +TNode<Smi> CodeStubAssembler::IntPtrOrSmiConstant<Smi>(int value) { + return SmiConstant(value); +} + +template <> +TNode<IntPtrT> CodeStubAssembler::IntPtrOrSmiConstant<IntPtrT>(int value) { + return IntPtrConstant(value); +} + +template <> +TNode<RawPtrT> CodeStubAssembler::IntPtrOrSmiConstant<RawPtrT>(int value) { + return ReinterpretCast<RawPtrT>(IntPtrConstant(value)); +} + Node* CodeStubAssembler::IntPtrOrSmiConstant(int value, ParameterMode mode) { if (mode == SMI_PARAMETERS) { return SmiConstant(value); @@ -273,41 +427,29 @@ Node* CodeStubAssembler::IntPtrOrSmiConstant(int value, ParameterMode mode) { } } -TNode<BoolT> CodeStubAssembler::IntPtrOrSmiEqual(Node* left, Node* right, - ParameterMode mode) { - if (mode == SMI_PARAMETERS) { - return SmiEqual(CAST(left), CAST(right)); - } else { - DCHECK_EQ(INTPTR_PARAMETERS, mode); - return IntPtrEqual(UncheckedCast<IntPtrT>(left), - UncheckedCast<IntPtrT>(right)); +bool CodeStubAssembler::IsIntPtrOrSmiConstantZero(TNode<Smi> test) { + Smi smi_test; + if (ToSmiConstant(test, &smi_test) && smi_test.value() == 0) { + return true; } + return false; } -TNode<BoolT> CodeStubAssembler::IntPtrOrSmiNotEqual(Node* left, Node* right, - ParameterMode mode) { - if (mode == SMI_PARAMETERS) { - return SmiNotEqual(CAST(left), CAST(right)); - } else { - DCHECK_EQ(INTPTR_PARAMETERS, mode); - return WordNotEqual(UncheckedCast<IntPtrT>(left), - UncheckedCast<IntPtrT>(right)); +bool CodeStubAssembler::IsIntPtrOrSmiConstantZero(TNode<IntPtrT> test) { + int32_t constant_test; + if (ToInt32Constant(test, &constant_test) && constant_test == 0) { + return true; } + return false; } bool CodeStubAssembler::IsIntPtrOrSmiConstantZero(Node* test, ParameterMode mode) { - int32_t constant_test; - Smi smi_test; if (mode == INTPTR_PARAMETERS) { - if (ToInt32Constant(test, &constant_test) && constant_test == 0) { - return true; - } + return IsIntPtrOrSmiConstantZero(UncheckedCast<IntPtrT>(test)); } else { DCHECK_EQ(mode, SMI_PARAMETERS); - if (ToSmiConstant(test, &smi_test) && smi_test.value() == 0) { - return true; - } + return IsIntPtrOrSmiConstantZero(UncheckedCast<Smi>(test)); } return false; } @@ -352,6 +494,10 @@ Node* CodeStubAssembler::MatchesParameterMode(Node* value, ParameterMode mode) { } TNode<BoolT> CodeStubAssembler::WordIsPowerOfTwo(SloppyTNode<IntPtrT> value) { + intptr_t constant; + if (ToIntPtrConstant(value, &constant)) { + return BoolConstant(base::bits::IsPowerOfTwo(constant)); + } // value && !(value & (value - 1)) return IntPtrEqual( Select<IntPtrT>( @@ -578,21 +724,44 @@ TNode<Float64T> CodeStubAssembler::Float64Trunc(SloppyTNode<Float64T> x) { TNode<BoolT> CodeStubAssembler::IsValidSmi(TNode<Smi> smi) { if (SmiValuesAre32Bits() && kSystemPointerSize == kInt64Size) { // Check that the Smi value is zero in the lower bits. - TNode<IntPtrT> value = BitcastTaggedSignedToWord(smi); + TNode<IntPtrT> value = BitcastTaggedToWordForTagAndSmiBits(smi); return Word32Equal(Int32Constant(0), TruncateIntPtrToInt32(value)); } return Int32TrueConstant(); } -Node* CodeStubAssembler::SmiShiftBitsConstant() { - return IntPtrConstant(kSmiShiftSize + kSmiTagSize); +TNode<BoolT> CodeStubAssembler::IsValidSmiIndex(TNode<Smi> smi) { + if (COMPRESS_POINTERS_BOOL) { + return WordEqual( + BitcastTaggedToWordForTagAndSmiBits(smi), + BitcastTaggedToWordForTagAndSmiBits(NormalizeSmiIndex(smi))); + } + return Int32TrueConstant(); +} + +TNode<Smi> CodeStubAssembler::NormalizeSmiIndex(TNode<Smi> smi_index) { + if (COMPRESS_POINTERS_BOOL) { + TNode<Int32T> raw = + TruncateWordToInt32(BitcastTaggedToWordForTagAndSmiBits(smi_index)); + smi_index = BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(raw)); + } + return smi_index; } TNode<Smi> CodeStubAssembler::SmiFromInt32(SloppyTNode<Int32T> value) { - TNode<IntPtrT> value_intptr = ChangeInt32ToIntPtr(value); - TNode<Smi> smi = - BitcastWordToTaggedSigned(WordShl(value_intptr, SmiShiftBitsConstant())); - return smi; + if (COMPRESS_POINTERS_BOOL) { + static_assert(!COMPRESS_POINTERS_BOOL || (kSmiShiftSize + kSmiTagSize == 1), + "Use shifting instead of add"); + return BitcastWordToTaggedSigned( + ChangeUint32ToWord(Int32Add(value, value))); + } + return SmiTag(ChangeInt32ToIntPtr(value)); +} + +TNode<Smi> CodeStubAssembler::SmiFromUint32(TNode<Uint32T> value) { + CSA_ASSERT(this, IntPtrLessThan(ChangeUint32ToWord(value), + IntPtrConstant(Smi::kMaxValue))); + return SmiFromInt32(Signed(value)); } TNode<BoolT> CodeStubAssembler::IsValidPositiveSmi(TNode<IntPtrT> value) { @@ -612,6 +781,9 @@ TNode<Smi> CodeStubAssembler::SmiTag(SloppyTNode<IntPtrT> value) { if (ToInt32Constant(value, &constant_value) && Smi::IsValid(constant_value)) { return SmiConstant(constant_value); } + if (COMPRESS_POINTERS_BOOL) { + return SmiFromInt32(TruncateIntPtrToInt32(value)); + } TNode<Smi> smi = BitcastWordToTaggedSigned(WordShl(value, SmiShiftBitsConstant())); return smi; @@ -622,11 +794,19 @@ TNode<IntPtrT> CodeStubAssembler::SmiUntag(SloppyTNode<Smi> value) { if (ToIntPtrConstant(value, &constant_value)) { return IntPtrConstant(constant_value >> (kSmiShiftSize + kSmiTagSize)); } - return Signed( - WordSar(BitcastTaggedSignedToWord(value), SmiShiftBitsConstant())); + if (COMPRESS_POINTERS_BOOL) { + return ChangeInt32ToIntPtr(SmiToInt32(value)); + } + return Signed(WordSar(BitcastTaggedToWordForTagAndSmiBits(value), + SmiShiftBitsConstant())); } TNode<Int32T> CodeStubAssembler::SmiToInt32(SloppyTNode<Smi> value) { + if (COMPRESS_POINTERS_BOOL) { + return Signed(Word32Sar( + TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(value)), + SmiShiftBitsConstant32())); + } TNode<IntPtrT> result = SmiUntag(value); return TruncateIntPtrToInt32(result); } @@ -673,13 +853,13 @@ TNode<Smi> CodeStubAssembler::TrySmiAdd(TNode<Smi> lhs, TNode<Smi> rhs, Label* if_overflow) { if (SmiValuesAre32Bits()) { return BitcastWordToTaggedSigned( - TryIntPtrAdd(BitcastTaggedSignedToWord(lhs), - BitcastTaggedSignedToWord(rhs), if_overflow)); + TryIntPtrAdd(BitcastTaggedToWordForTagAndSmiBits(lhs), + BitcastTaggedToWordForTagAndSmiBits(rhs), if_overflow)); } else { DCHECK(SmiValuesAre31Bits()); TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow( - TruncateIntPtrToInt32(BitcastTaggedSignedToWord(lhs)), - TruncateIntPtrToInt32(BitcastTaggedSignedToWord(rhs))); + TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(lhs)), + TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(rhs))); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, if_overflow); TNode<Int32T> result = Projection<0>(pair); @@ -690,8 +870,9 @@ TNode<Smi> CodeStubAssembler::TrySmiAdd(TNode<Smi> lhs, TNode<Smi> rhs, TNode<Smi> CodeStubAssembler::TrySmiSub(TNode<Smi> lhs, TNode<Smi> rhs, Label* if_overflow) { if (SmiValuesAre32Bits()) { - TNode<PairT<IntPtrT, BoolT>> pair = IntPtrSubWithOverflow( - BitcastTaggedSignedToWord(lhs), BitcastTaggedSignedToWord(rhs)); + TNode<PairT<IntPtrT, BoolT>> pair = + IntPtrSubWithOverflow(BitcastTaggedToWordForTagAndSmiBits(lhs), + BitcastTaggedToWordForTagAndSmiBits(rhs)); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, if_overflow); TNode<IntPtrT> result = Projection<0>(pair); @@ -699,8 +880,8 @@ TNode<Smi> CodeStubAssembler::TrySmiSub(TNode<Smi> lhs, TNode<Smi> rhs, } else { DCHECK(SmiValuesAre31Bits()); TNode<PairT<Int32T, BoolT>> pair = Int32SubWithOverflow( - TruncateIntPtrToInt32(BitcastTaggedSignedToWord(lhs)), - TruncateIntPtrToInt32(BitcastTaggedSignedToWord(rhs))); + TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(lhs)), + TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(rhs))); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, if_overflow); TNode<Int32T> result = Projection<0>(pair); @@ -878,7 +1059,7 @@ TNode<Number> CodeStubAssembler::SmiMul(TNode<Smi> a, TNode<Smi> b) { } BIND(&answer_zero); { - TNode<Word32T> or_result = Word32Or(lhs32, rhs32); + TNode<Int32T> or_result = Word32Or(lhs32, rhs32); Label if_should_be_negative_zero(this), if_should_be_zero(this); Branch(Int32LessThan(or_result, zero), &if_should_be_negative_zero, &if_should_be_zero); @@ -982,41 +1163,27 @@ TNode<Int32T> CodeStubAssembler::TruncateIntPtrToInt32( return ReinterpretCast<Int32T>(value); } -TNode<BoolT> CodeStubAssembler::TaggedIsSmi(SloppyTNode<Object> a) { - STATIC_ASSERT(kSmiTagMask < kMaxUInt32); - return Word32Equal(Word32And(TruncateIntPtrToInt32(BitcastTaggedToWord(a)), - Int32Constant(kSmiTagMask)), - Int32Constant(0)); -} - TNode<BoolT> CodeStubAssembler::TaggedIsSmi(TNode<MaybeObject> a) { STATIC_ASSERT(kSmiTagMask < kMaxUInt32); return Word32Equal( - Word32And(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(a)), + Word32And(TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)), Int32Constant(kSmiTagMask)), Int32Constant(0)); } -TNode<BoolT> CodeStubAssembler::TaggedIsNotSmi(SloppyTNode<Object> a) { - // Although BitcastTaggedSignedToWord is generally unsafe on HeapObjects, we - // can nonetheless use it to inspect the Smi tag. The assumption here is that - // the GC will not exchange Smis for HeapObjects or vice-versa. - TNode<IntPtrT> a_bitcast = BitcastTaggedSignedToWord(UncheckedCast<Smi>(a)); - STATIC_ASSERT(kSmiTagMask < kMaxUInt32); - return Word32NotEqual( - Word32And(TruncateIntPtrToInt32(a_bitcast), Int32Constant(kSmiTagMask)), - Int32Constant(0)); +TNode<BoolT> CodeStubAssembler::TaggedIsNotSmi(TNode<MaybeObject> a) { + return Word32BinaryNot(TaggedIsSmi(a)); } TNode<BoolT> CodeStubAssembler::TaggedIsPositiveSmi(SloppyTNode<Object> a) { #if defined(V8_HOST_ARCH_32_BIT) || defined(V8_31BIT_SMIS_ON_64BIT_ARCH) return Word32Equal( Word32And( - TruncateIntPtrToInt32(BitcastTaggedToWord(a)), - Uint32Constant(kSmiTagMask | static_cast<int32_t>(kSmiSignMask))), + TruncateIntPtrToInt32(BitcastTaggedToWordForTagAndSmiBits(a)), + Uint32Constant(static_cast<uint32_t>(kSmiTagMask | kSmiSignMask))), Int32Constant(0)); #else - return WordEqual(WordAnd(BitcastTaggedToWord(a), + return WordEqual(WordAnd(BitcastTaggedToWordForTagAndSmiBits(a), IntPtrConstant(kSmiTagMask | kSmiSignMask)), IntPtrConstant(0)); #endif @@ -1052,55 +1219,6 @@ TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck( INTPTR_PARAMETERS, if_hole); } -void CodeStubAssembler::BranchIfPrototypesHaveNoElements( - Node* receiver_map, Label* definitely_no_elements, - Label* possibly_elements) { - CSA_SLOW_ASSERT(this, IsMap(receiver_map)); - VARIABLE(var_map, MachineRepresentation::kTagged, receiver_map); - Label loop_body(this, &var_map); - TNode<FixedArray> empty_fixed_array = EmptyFixedArrayConstant(); - TNode<NumberDictionary> empty_slow_element_dictionary = - EmptySlowElementDictionaryConstant(); - Goto(&loop_body); - - BIND(&loop_body); - { - Node* map = var_map.value(); - TNode<HeapObject> prototype = LoadMapPrototype(map); - GotoIf(IsNull(prototype), definitely_no_elements); - TNode<Map> prototype_map = LoadMap(prototype); - TNode<Uint16T> prototype_instance_type = LoadMapInstanceType(prototype_map); - - // Pessimistically assume elements if a Proxy, Special API Object, - // or JSPrimitiveWrapper wrapper is found on the prototype chain. After this - // instance type check, it's not necessary to check for interceptors or - // access checks. - Label if_custom(this, Label::kDeferred), if_notcustom(this); - Branch(IsCustomElementsReceiverInstanceType(prototype_instance_type), - &if_custom, &if_notcustom); - - BIND(&if_custom); - { - // For string JSPrimitiveWrapper wrappers we still support the checks as - // long as they wrap the empty string. - GotoIfNot( - InstanceTypeEqual(prototype_instance_type, JS_PRIMITIVE_WRAPPER_TYPE), - possibly_elements); - Node* prototype_value = LoadJSPrimitiveWrapperValue(prototype); - Branch(IsEmptyString(prototype_value), &if_notcustom, possibly_elements); - } - - BIND(&if_notcustom); - { - TNode<FixedArrayBase> prototype_elements = LoadElements(CAST(prototype)); - var_map.Bind(prototype_map); - GotoIf(TaggedEqual(prototype_elements, empty_fixed_array), &loop_body); - Branch(TaggedEqual(prototype_elements, empty_slow_element_dictionary), - &loop_body, possibly_elements); - } - } -} - void CodeStubAssembler::BranchIfJSReceiver(SloppyTNode<Object> object, Label* if_true, Label* if_false) { GotoIf(TaggedIsSmi(object), if_false); @@ -1118,19 +1236,6 @@ void CodeStubAssembler::GotoIfForceSlowPath(Label* if_true) { #endif } -void CodeStubAssembler::GotoIfDebugExecutionModeChecksSideEffects( - Label* if_true) { - STATIC_ASSERT(sizeof(DebugInfo::ExecutionMode) >= sizeof(int32_t)); - - TNode<ExternalReference> execution_mode_address = ExternalConstant( - ExternalReference::debug_execution_mode_address(isolate())); - TNode<Int32T> execution_mode = - UncheckedCast<Int32T>(Load(MachineType::Int32(), execution_mode_address)); - - GotoIf(Word32Equal(execution_mode, Int32Constant(DebugInfo::kSideEffects)), - if_true); -} - TNode<HeapObject> CodeStubAssembler::AllocateRaw(TNode<IntPtrT> size_in_bytes, AllocationFlags flags, TNode<RawPtrT> top_address, @@ -1557,7 +1662,7 @@ void CodeStubAssembler::GotoIfMapHasSlowProperties(TNode<Map> map, } TNode<HeapObject> CodeStubAssembler::LoadFastProperties( - SloppyTNode<JSObject> object) { + SloppyTNode<JSReceiver> object) { CSA_SLOW_ASSERT(this, Word32BinaryNot(IsDictionaryMap(LoadMap(object)))); TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object); return Select<HeapObject>( @@ -1566,7 +1671,7 @@ TNode<HeapObject> CodeStubAssembler::LoadFastProperties( } TNode<HeapObject> CodeStubAssembler::LoadSlowProperties( - SloppyTNode<JSObject> object) { + SloppyTNode<JSReceiver> object) { CSA_SLOW_ASSERT(this, IsDictionaryMap(LoadMap(object))); TNode<Object> properties = LoadJSReceiverPropertiesOrHash(object); return Select<HeapObject>( @@ -1862,18 +1967,8 @@ TNode<Uint32T> CodeStubAssembler::LoadStringLengthAsWord32( return LoadObjectField<Uint32T>(string, String::kLengthOffset); } -Node* CodeStubAssembler::PointerToSeqStringData(Node* seq_string) { - CSA_ASSERT(this, IsString(seq_string)); - CSA_ASSERT(this, - IsSequentialStringInstanceType(LoadInstanceType(seq_string))); - STATIC_ASSERT(SeqOneByteString::kHeaderSize == SeqTwoByteString::kHeaderSize); - return IntPtrAdd( - BitcastTaggedToWord(seq_string), - IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag)); -} - -Node* CodeStubAssembler::LoadJSPrimitiveWrapperValue(Node* object) { - CSA_ASSERT(this, IsJSPrimitiveWrapper(object)); +TNode<Object> CodeStubAssembler::LoadJSPrimitiveWrapperValue( + TNode<JSPrimitiveWrapper> object) { return LoadObjectField(object, JSPrimitiveWrapper::kValueOffset); } @@ -1887,15 +1982,9 @@ void CodeStubAssembler::DispatchMaybeObject(TNode<MaybeObject> maybe_object, GotoIf(IsCleared(maybe_object), if_cleared); - GotoIf(Word32Equal(Word32And(TruncateIntPtrToInt32( - BitcastMaybeObjectToWord(maybe_object)), - Int32Constant(kHeapObjectTagMask)), - Int32Constant(kHeapObjectTag)), - &inner_if_strong); + GotoIf(IsStrong(maybe_object), &inner_if_strong); - *extracted = - BitcastWordToTagged(WordAnd(BitcastMaybeObjectToWord(maybe_object), - IntPtrConstant(~kWeakHeapObjectMask))); + *extracted = GetHeapObjectAssumeWeak(maybe_object); Goto(if_weak); BIND(&inner_if_smi); @@ -1908,10 +1997,10 @@ void CodeStubAssembler::DispatchMaybeObject(TNode<MaybeObject> maybe_object, } TNode<BoolT> CodeStubAssembler::IsStrong(TNode<MaybeObject> value) { - return Word32Equal( - Word32And(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)), - Int32Constant(kHeapObjectTagMask)), - Int32Constant(kHeapObjectTag)); + return Word32Equal(Word32And(TruncateIntPtrToInt32( + BitcastTaggedToWordForTagAndSmiBits(value)), + Int32Constant(kHeapObjectTagMask)), + Int32Constant(kHeapObjectTag)); } TNode<HeapObject> CodeStubAssembler::GetHeapObjectIfStrong( @@ -1921,10 +2010,10 @@ TNode<HeapObject> CodeStubAssembler::GetHeapObjectIfStrong( } TNode<BoolT> CodeStubAssembler::IsWeakOrCleared(TNode<MaybeObject> value) { - return Word32Equal( - Word32And(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)), - Int32Constant(kHeapObjectTagMask)), - Int32Constant(kWeakHeapObjectTag)); + return Word32Equal(Word32And(TruncateIntPtrToInt32( + BitcastTaggedToWordForTagAndSmiBits(value)), + Int32Constant(kHeapObjectTagMask)), + Int32Constant(kWeakHeapObjectTag)); } TNode<BoolT> CodeStubAssembler::IsCleared(TNode<MaybeObject> value) { @@ -1932,11 +2021,6 @@ TNode<BoolT> CodeStubAssembler::IsCleared(TNode<MaybeObject> value) { Int32Constant(kClearedWeakHeapObjectLower32)); } -TNode<BoolT> CodeStubAssembler::IsNotCleared(TNode<MaybeObject> value) { - return Word32NotEqual(TruncateIntPtrToInt32(BitcastMaybeObjectToWord(value)), - Int32Constant(kClearedWeakHeapObjectLower32)); -} - TNode<HeapObject> CodeStubAssembler::GetHeapObjectAssumeWeak( TNode<MaybeObject> value) { CSA_ASSERT(this, IsWeakOrCleared(value)); @@ -1951,43 +2035,41 @@ TNode<HeapObject> CodeStubAssembler::GetHeapObjectAssumeWeak( return GetHeapObjectAssumeWeak(value); } -TNode<BoolT> CodeStubAssembler::IsWeakReferenceTo(TNode<MaybeObject> object, - TNode<Object> value) { -#if defined(V8_HOST_ARCH_32_BIT) || defined(V8_COMPRESS_POINTERS) - STATIC_ASSERT(kTaggedSize == kInt32Size); - return Word32Equal( - Word32And(TruncateWordToInt32(BitcastMaybeObjectToWord(object)), - Uint32Constant( - static_cast<uint32_t>(~kWeakHeapObjectMask & kMaxUInt32))), - TruncateWordToInt32(BitcastTaggedToWord(value))); -#else - return WordEqual(WordAnd(BitcastMaybeObjectToWord(object), - IntPtrConstant(~kWeakHeapObjectMask)), - BitcastTaggedToWord(value)); - -#endif -} - -TNode<BoolT> CodeStubAssembler::IsStrongReferenceTo(TNode<MaybeObject> object, - TNode<Object> value) { - return TaggedEqual(BitcastWordToTagged(BitcastMaybeObjectToWord(object)), - value); -} - -TNode<BoolT> CodeStubAssembler::IsNotWeakReferenceTo(TNode<MaybeObject> object, - TNode<Object> value) { -#if defined(V8_HOST_ARCH_32_BIT) || defined(V8_COMPRESS_POINTERS) - return Word32NotEqual( - Word32And(TruncateWordToInt32(BitcastMaybeObjectToWord(object)), - Uint32Constant( - static_cast<uint32_t>(~kWeakHeapObjectMask & kMaxUInt32))), - TruncateWordToInt32(BitcastTaggedToWord(value))); -#else - return WordNotEqual(WordAnd(BitcastMaybeObjectToWord(object), - IntPtrConstant(~kWeakHeapObjectMask)), - BitcastTaggedToWord(value)); - -#endif +// This version generates +// (maybe_object & ~mask) == value +// It works for non-Smi |maybe_object| and for both Smi and HeapObject values +// but requires a big constant for ~mask. +TNode<BoolT> CodeStubAssembler::IsWeakReferenceToObject( + TNode<MaybeObject> maybe_object, TNode<Object> value) { + CSA_ASSERT(this, TaggedIsNotSmi(maybe_object)); + if (COMPRESS_POINTERS_BOOL) { + return Word32Equal( + Word32And(TruncateWordToInt32(BitcastMaybeObjectToWord(maybe_object)), + Uint32Constant(~static_cast<uint32_t>(kWeakHeapObjectMask))), + TruncateWordToInt32(BitcastTaggedToWord(value))); + } else { + return WordEqual(WordAnd(BitcastMaybeObjectToWord(maybe_object), + IntPtrConstant(~kWeakHeapObjectMask)), + BitcastTaggedToWord(value)); + } +} + +// This version generates +// maybe_object == (heap_object | mask) +// It works for any |maybe_object| values and generates a better code because it +// uses a small constant for mask. +TNode<BoolT> CodeStubAssembler::IsWeakReferenceTo( + TNode<MaybeObject> maybe_object, TNode<HeapObject> heap_object) { + if (COMPRESS_POINTERS_BOOL) { + return Word32Equal( + TruncateWordToInt32(BitcastMaybeObjectToWord(maybe_object)), + Word32Or(TruncateWordToInt32(BitcastTaggedToWord(heap_object)), + Int32Constant(kWeakHeapObjectMask))); + } else { + return WordEqual(BitcastMaybeObjectToWord(maybe_object), + WordOr(BitcastTaggedToWord(heap_object), + IntPtrConstant(kWeakHeapObjectMask))); + } } TNode<MaybeObject> CodeStubAssembler::MakeWeak(TNode<HeapObject> value) { @@ -2123,16 +2205,27 @@ TNode<IntPtrT> CodeStubAssembler::LoadPropertyArrayLength( return Signed(DecodeWord<PropertyArray::LengthField>(value)); } -TNode<RawPtrT> CodeStubAssembler::LoadJSTypedArrayBackingStore( +TNode<RawPtrT> CodeStubAssembler::LoadJSTypedArrayDataPtr( TNode<JSTypedArray> typed_array) { - // Backing store = external_pointer + base_pointer. - Node* external_pointer = - LoadObjectField(typed_array, JSTypedArray::kExternalPointerOffset, - MachineType::Pointer()); - TNode<Object> base_pointer = - LoadObjectField(typed_array, JSTypedArray::kBasePointerOffset); - return UncheckedCast<RawPtrT>( - IntPtrAdd(external_pointer, BitcastTaggedToWord(base_pointer))); + // Data pointer = external_pointer + static_cast<Tagged_t>(base_pointer). + TNode<RawPtrT> external_pointer = LoadObjectField<RawPtrT>( + typed_array, JSTypedArray::kExternalPointerOffset); + + TNode<IntPtrT> base_pointer; + if (COMPRESS_POINTERS_BOOL) { + TNode<Int32T> compressed_base = + LoadObjectField<Int32T>(typed_array, JSTypedArray::kBasePointerOffset); + // Zero-extend TaggedT to WordT according to current compression scheme + // so that the addition with |external_pointer| (which already contains + // compensated offset value) below will decompress the tagged value. + // See JSTypedArray::ExternalPointerCompensationForOnHeapArray() for + // details. + base_pointer = Signed(ChangeUint32ToWord(compressed_base)); + } else { + base_pointer = + LoadObjectField<IntPtrT>(typed_array, JSTypedArray::kBasePointerOffset); + } + return RawPtrAdd(external_pointer, base_pointer); } TNode<BigInt> CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged( @@ -2267,8 +2360,7 @@ TNode<BigInt> CodeStubAssembler::BigIntFromInt64(TNode<IntPtrT> value) { return var_result.value(); } -compiler::TNode<BigInt> -CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged( +TNode<BigInt> CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged( SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset) { Label if_zero(this), done(this); if (Is64()) { @@ -2416,59 +2508,30 @@ TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( return var_result.value(); } -void CodeStubAssembler::StoreJSTypedArrayElementFromTagged( - TNode<Context> context, TNode<JSTypedArray> typed_array, - TNode<Smi> index_node, TNode<Object> value, ElementsKind elements_kind) { - TNode<RawPtrT> data_pointer = LoadJSTypedArrayBackingStore(typed_array); - switch (elements_kind) { - case UINT8_ELEMENTS: - case UINT8_CLAMPED_ELEMENTS: - case INT8_ELEMENTS: - case UINT16_ELEMENTS: - case INT16_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, - SmiToInt32(CAST(value)), SMI_PARAMETERS); - break; - case UINT32_ELEMENTS: - case INT32_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, - TruncateTaggedToWord32(context, value), SMI_PARAMETERS); - break; - case FLOAT32_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, - TruncateFloat64ToFloat32(LoadHeapNumberValue(CAST(value))), - SMI_PARAMETERS); - break; - case FLOAT64_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, - LoadHeapNumberValue(CAST(value)), SMI_PARAMETERS); - break; - case BIGUINT64_ELEMENTS: - case BIGINT64_ELEMENTS: - StoreElement(data_pointer, elements_kind, index_node, - UncheckedCast<BigInt>(value), SMI_PARAMETERS); - break; - default: - UNREACHABLE(); - } -} - +template <typename TIndex> TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot( - Node* object, Node* slot_index_node, int additional_offset, - ParameterMode parameter_mode) { - CSA_SLOW_ASSERT(this, IsFeedbackVector(object)); - CSA_SLOW_ASSERT(this, MatchesParameterMode(slot_index_node, parameter_mode)); + TNode<FeedbackVector> feedback_vector, TNode<TIndex> slot, + int additional_offset) { int32_t header_size = FeedbackVector::kFeedbackSlotsOffset + additional_offset - kHeapObjectTag; - TNode<IntPtrT> offset = ElementOffsetFromIndex( - slot_index_node, HOLEY_ELEMENTS, parameter_mode, header_size); + TNode<IntPtrT> offset = + ElementOffsetFromIndex(slot, HOLEY_ELEMENTS, header_size); CSA_SLOW_ASSERT( - this, IsOffsetInBounds(offset, LoadFeedbackVectorLength(CAST(object)), + this, IsOffsetInBounds(offset, LoadFeedbackVectorLength(feedback_vector), FeedbackVector::kHeaderSize)); - return UncheckedCast<MaybeObject>( - Load(MachineType::AnyTagged(), object, offset)); + return Load<MaybeObject>(feedback_vector, offset); } +template TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot( + TNode<FeedbackVector> feedback_vector, TNode<Smi> slot, + int additional_offset); +template TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot( + TNode<FeedbackVector> feedback_vector, TNode<IntPtrT> slot, + int additional_offset); +template TNode<MaybeObject> CodeStubAssembler::LoadFeedbackVectorSlot( + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot, + int additional_offset); + template <typename Array> TNode<Int32T> CodeStubAssembler::LoadAndUntagToWord32ArrayElement( TNode<Array> object, int array_header_size, Node* index_node, @@ -2617,6 +2680,13 @@ TNode<Float64T> CodeStubAssembler::LoadDoubleWithHoleCheck( return UncheckedCast<Float64T>(Load(machine_type, base, offset)); } +TNode<BoolT> CodeStubAssembler::LoadContextHasExtensionField( + SloppyTNode<Context> context) { + TNode<IntPtrT> value = + LoadAndUntagObjectField(context, Context::kLengthOffset); + return IsSetWord<Context::HasExtensionField>(value); +} + TNode<Object> CodeStubAssembler::LoadContextElement( SloppyTNode<Context> context, int slot_index) { int offset = Context::SlotOffset(slot_index); @@ -2626,15 +2696,15 @@ TNode<Object> CodeStubAssembler::LoadContextElement( TNode<Object> CodeStubAssembler::LoadContextElement( SloppyTNode<Context> context, SloppyTNode<IntPtrT> slot_index) { - TNode<IntPtrT> offset = ElementOffsetFromIndex( - slot_index, PACKED_ELEMENTS, INTPTR_PARAMETERS, Context::SlotOffset(0)); + TNode<IntPtrT> offset = ElementOffsetFromIndex(slot_index, PACKED_ELEMENTS, + Context::SlotOffset(0)); return UncheckedCast<Object>(Load(MachineType::AnyTagged(), context, offset)); } TNode<Object> CodeStubAssembler::LoadContextElement(TNode<Context> context, TNode<Smi> slot_index) { - TNode<IntPtrT> offset = ElementOffsetFromIndex( - slot_index, PACKED_ELEMENTS, SMI_PARAMETERS, Context::SlotOffset(0)); + TNode<IntPtrT> offset = ElementOffsetFromIndex(slot_index, PACKED_ELEMENTS, + Context::SlotOffset(0)); return UncheckedCast<Object>(Load(MachineType::AnyTagged(), context, offset)); } @@ -2949,33 +3019,30 @@ void CodeStubAssembler::StoreFixedDoubleArrayElement( StoreNoWriteBarrier(rep, object, offset, value_silenced); } -void CodeStubAssembler::StoreFeedbackVectorSlot(Node* object, - Node* slot_index_node, - Node* value, - WriteBarrierMode barrier_mode, - int additional_offset, - ParameterMode parameter_mode) { - CSA_SLOW_ASSERT(this, IsFeedbackVector(object)); - CSA_SLOW_ASSERT(this, MatchesParameterMode(slot_index_node, parameter_mode)); +void CodeStubAssembler::StoreFeedbackVectorSlot( + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot, + TNode<AnyTaggedT> value, WriteBarrierMode barrier_mode, + int additional_offset) { DCHECK(IsAligned(additional_offset, kTaggedSize)); DCHECK(barrier_mode == SKIP_WRITE_BARRIER || barrier_mode == UNSAFE_SKIP_WRITE_BARRIER || barrier_mode == UPDATE_WRITE_BARRIER); int header_size = FeedbackVector::kFeedbackSlotsOffset + additional_offset - kHeapObjectTag; - TNode<IntPtrT> offset = ElementOffsetFromIndex( - slot_index_node, HOLEY_ELEMENTS, parameter_mode, header_size); - // Check that slot_index_node <= object.length. + TNode<IntPtrT> offset = + ElementOffsetFromIndex(Signed(slot), HOLEY_ELEMENTS, header_size); + // Check that slot <= feedback_vector.length. CSA_ASSERT(this, - IsOffsetInBounds(offset, LoadFeedbackVectorLength(CAST(object)), + IsOffsetInBounds(offset, LoadFeedbackVectorLength(feedback_vector), FeedbackVector::kHeaderSize)); if (barrier_mode == SKIP_WRITE_BARRIER) { - StoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, value); + StoreNoWriteBarrier(MachineRepresentation::kTagged, feedback_vector, offset, + value); } else if (barrier_mode == UNSAFE_SKIP_WRITE_BARRIER) { - UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged, object, offset, - value); + UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged, feedback_vector, + offset, value); } else { - Store(object, offset, value); + Store(feedback_vector, offset, value); } } @@ -3045,33 +3112,29 @@ TNode<Smi> CodeStubAssembler::BuildAppendJSArray(ElementsKind kind, Label success(this); TVARIABLE(Smi, var_tagged_length); ParameterMode mode = OptimalParameterMode(); - VARIABLE(var_length, OptimalParameterRepresentation(), - TaggedToParameter(LoadFastJSArrayLength(array), mode)); - VARIABLE(var_elements, MachineRepresentation::kTagged, LoadElements(array)); + TVARIABLE(BInt, var_length, SmiToBInt(LoadFastJSArrayLength(array))); + TVARIABLE(FixedArrayBase, var_elements, LoadElements(array)); // Resize the capacity of the fixed array if it doesn't fit. TNode<IntPtrT> first = arg_index->value(); - Node* growth = IntPtrToParameter( - IntPtrSub(UncheckedCast<IntPtrT>(args->GetLength(INTPTR_PARAMETERS)), - first), - mode); + TNode<BInt> growth = IntPtrToBInt(IntPtrSub(args->GetLength(), first)); PossiblyGrowElementsCapacity(mode, kind, array, var_length.value(), &var_elements, growth, &pre_bailout); // Push each argument onto the end of the array now that there is enough // capacity. CodeStubAssembler::VariableList push_vars({&var_length}, zone()); - Node* elements = var_elements.value(); + TNode<FixedArrayBase> elements = var_elements.value(); args->ForEach( push_vars, - [this, kind, mode, elements, &var_length, &pre_bailout](Node* arg) { + [&](TNode<Object> arg) { TryStoreArrayElement(kind, mode, &pre_bailout, elements, var_length.value(), arg); - Increment(&var_length, 1, mode); + Increment(&var_length); }, - first, nullptr); + first); { - TNode<Smi> length = ParameterToTagged(var_length.value(), mode); + TNode<Smi> length = BIntToSmi(var_length.value()); var_tagged_length = length; StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length); Goto(&success); @@ -3111,8 +3174,7 @@ void CodeStubAssembler::BuildAppendJSArray(ElementsKind kind, Node* array, CSA_SLOW_ASSERT(this, IsJSArray(array)); Comment("BuildAppendJSArray: ", ElementsKindToString(kind)); ParameterMode mode = OptimalParameterMode(); - VARIABLE(var_length, OptimalParameterRepresentation(), - TaggedToParameter(LoadFastJSArrayLength(array), mode)); + TVARIABLE(BInt, var_length, SmiToBInt(LoadFastJSArrayLength(array))); VARIABLE(var_elements, MachineRepresentation::kTagged, LoadElements(array)); // Resize the capacity of the fixed array if it doesn't fit. @@ -3124,9 +3186,9 @@ void CodeStubAssembler::BuildAppendJSArray(ElementsKind kind, Node* array, // capacity. TryStoreArrayElement(kind, mode, bailout, var_elements.value(), var_length.value(), value); - Increment(&var_length, 1, mode); + Increment(&var_length); - TNode<Smi> length = ParameterToTagged(var_length.value(), mode); + TNode<Smi> length = BIntToSmi(var_length.value()); StoreObjectFieldNoWriteBarrier(array, JSArray::kLengthOffset, length); } @@ -3138,7 +3200,7 @@ Node* CodeStubAssembler::AllocateCellWithValue(Node* value, return result; } -Node* CodeStubAssembler::LoadCellValue(Node* cell) { +TNode<Object> CodeStubAssembler::LoadCellValue(Node* cell) { CSA_SLOW_ASSERT(this, HasInstanceType(cell, CELL_TYPE)); return LoadObjectField(cell, Cell::kValueOffset); } @@ -3278,7 +3340,8 @@ TNode<ByteArray> CodeStubAssembler::AllocateByteArray(TNode<UintPtrT> length, TNode<IntPtrT> raw_size = GetArrayAllocationSize(Signed(length), UINT8_ELEMENTS, INTPTR_PARAMETERS, ByteArray::kHeaderSize + kObjectAlignmentMask); - TNode<WordT> size = WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); + TNode<IntPtrT> size = + WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); Branch(IntPtrLessThanOrEqual(size, IntPtrConstant(kMaxRegularHeapObjectSize)), &if_sizeissmall, &if_notsizeissmall); @@ -3352,7 +3415,8 @@ TNode<String> CodeStubAssembler::AllocateSeqOneByteString( TNode<IntPtrT> raw_size = GetArrayAllocationSize( Signed(ChangeUint32ToWord(length)), UINT8_ELEMENTS, INTPTR_PARAMETERS, SeqOneByteString::kHeaderSize + kObjectAlignmentMask); - TNode<WordT> size = WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); + TNode<IntPtrT> size = + WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); Branch(IntPtrLessThanOrEqual(size, IntPtrConstant(kMaxRegularHeapObjectSize)), &if_sizeissmall, &if_notsizeissmall); @@ -3423,7 +3487,8 @@ TNode<String> CodeStubAssembler::AllocateSeqTwoByteString( TNode<IntPtrT> raw_size = GetArrayAllocationSize( Signed(ChangeUint32ToWord(length)), UINT16_ELEMENTS, INTPTR_PARAMETERS, SeqOneByteString::kHeaderSize + kObjectAlignmentMask); - TNode<WordT> size = WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); + TNode<IntPtrT> size = + WordAnd(raw_size, IntPtrConstant(~kObjectAlignmentMask)); Branch(IntPtrLessThanOrEqual(size, IntPtrConstant(kMaxRegularHeapObjectSize)), &if_sizeissmall, &if_notsizeissmall); @@ -3496,35 +3561,6 @@ TNode<String> CodeStubAssembler::AllocateSlicedTwoByteString( offset); } -TNode<String> CodeStubAssembler::AllocateConsString(TNode<Uint32T> length, - TNode<String> left, - TNode<String> right) { - // Added string can be a cons string. - Comment("Allocating ConsString"); - TNode<Int32T> left_instance_type = LoadInstanceType(left); - TNode<Int32T> right_instance_type = LoadInstanceType(right); - - // Determine the resulting ConsString map to use depending on whether - // any of {left} or {right} has two byte encoding. - STATIC_ASSERT(kOneByteStringTag != 0); - STATIC_ASSERT(kTwoByteStringTag == 0); - TNode<Int32T> combined_instance_type = - Word32And(left_instance_type, right_instance_type); - TNode<Map> result_map = CAST(Select<Object>( - IsSetWord32(combined_instance_type, kStringEncodingMask), - [=] { return ConsOneByteStringMapConstant(); }, - [=] { return ConsStringMapConstant(); })); - TNode<HeapObject> result = AllocateInNewSpace(ConsString::kSize); - StoreMapNoWriteBarrier(result, result_map); - StoreObjectFieldNoWriteBarrier(result, ConsString::kLengthOffset, length, - MachineRepresentation::kWord32); - StoreObjectFieldNoWriteBarrier(result, ConsString::kHashFieldOffset, - Int32Constant(String::kEmptyHashField), - MachineRepresentation::kWord32); - StoreObjectFieldNoWriteBarrier(result, ConsString::kFirstOffset, left); - StoreObjectFieldNoWriteBarrier(result, ConsString::kSecondOffset, right); - return CAST(result); -} TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary( int at_least_space_for) { @@ -3762,106 +3798,26 @@ template V8_EXPORT_PRIVATE TNode<SmallOrderedHashSet> CodeStubAssembler::AllocateSmallOrderedHashTable<SmallOrderedHashSet>( TNode<IntPtrT> capacity); -template <typename CollectionType> -void CodeStubAssembler::FindOrderedHashTableEntry( - Node* table, Node* hash, - const std::function<void(TNode<Object>, Label*, Label*)>& key_compare, - Variable* entry_start_position, Label* entry_found, Label* not_found) { - // Get the index of the bucket. - TNode<IntPtrT> const number_of_buckets = - SmiUntag(CAST(UnsafeLoadFixedArrayElement( - CAST(table), CollectionType::NumberOfBucketsIndex()))); - TNode<WordT> const bucket = - WordAnd(hash, IntPtrSub(number_of_buckets, IntPtrConstant(1))); - TNode<IntPtrT> const first_entry = SmiUntag(CAST(UnsafeLoadFixedArrayElement( - CAST(table), bucket, - CollectionType::HashTableStartIndex() * kTaggedSize))); - - // Walk the bucket chain. - TNode<IntPtrT> entry_start; - Label if_key_found(this); - { - TVARIABLE(IntPtrT, var_entry, first_entry); - Label loop(this, {&var_entry, entry_start_position}), - continue_next_entry(this); - Goto(&loop); - BIND(&loop); - - // If the entry index is the not-found sentinel, we are done. - GotoIf(IntPtrEqual(var_entry.value(), - IntPtrConstant(CollectionType::kNotFound)), - not_found); - - // Make sure the entry index is within range. - CSA_ASSERT( - this, - UintPtrLessThan( - var_entry.value(), - SmiUntag(SmiAdd( - CAST(UnsafeLoadFixedArrayElement( - CAST(table), CollectionType::NumberOfElementsIndex())), - CAST(UnsafeLoadFixedArrayElement( - CAST(table), - CollectionType::NumberOfDeletedElementsIndex())))))); - - // Compute the index of the entry relative to kHashTableStartIndex. - entry_start = - IntPtrAdd(IntPtrMul(var_entry.value(), - IntPtrConstant(CollectionType::kEntrySize)), - number_of_buckets); - - // Load the key from the entry. - TNode<Object> const candidate_key = UnsafeLoadFixedArrayElement( - CAST(table), entry_start, - CollectionType::HashTableStartIndex() * kTaggedSize); - - key_compare(candidate_key, &if_key_found, &continue_next_entry); - - BIND(&continue_next_entry); - // Load the index of the next entry in the bucket chain. - var_entry = SmiUntag(CAST(UnsafeLoadFixedArrayElement( - CAST(table), entry_start, - (CollectionType::HashTableStartIndex() + CollectionType::kChainOffset) * - kTaggedSize))); - - Goto(&loop); - } - - BIND(&if_key_found); - entry_start_position->Bind(entry_start); - Goto(entry_found); -} - -template void CodeStubAssembler::FindOrderedHashTableEntry<OrderedHashMap>( - Node* table, Node* hash, - const std::function<void(TNode<Object>, Label*, Label*)>& key_compare, - Variable* entry_start_position, Label* entry_found, Label* not_found); -template void CodeStubAssembler::FindOrderedHashTableEntry<OrderedHashSet>( - Node* table, Node* hash, - const std::function<void(TNode<Object>, Label*, Label*)>& key_compare, - Variable* entry_start_position, Label* entry_found, Label* not_found); - Node* CodeStubAssembler::AllocateStruct(Node* map, AllocationFlags flags) { Comment("AllocateStruct"); CSA_ASSERT(this, IsMap(map)); TNode<IntPtrT> size = TimesTaggedSize(LoadMapInstanceSizeInWords(map)); TNode<HeapObject> object = Allocate(size, flags); StoreMapNoWriteBarrier(object, map); - InitializeStructBody(object, map, size, Struct::kHeaderSize); + InitializeStructBody(object, size, Struct::kHeaderSize); return object; } -void CodeStubAssembler::InitializeStructBody(Node* object, Node* map, - Node* size, int start_offset) { - CSA_SLOW_ASSERT(this, IsMap(map)); +void CodeStubAssembler::InitializeStructBody(TNode<HeapObject> object, + TNode<IntPtrT> size, + int start_offset) { Comment("InitializeStructBody"); TNode<Oddball> filler = UndefinedConstant(); // Calculate the untagged field addresses. - object = BitcastTaggedToWord(object); - TNode<WordT> start_address = - IntPtrAdd(object, IntPtrConstant(start_offset - kHeapObjectTag)); - TNode<WordT> end_address = - IntPtrSub(IntPtrAdd(object, size), IntPtrConstant(kHeapObjectTag)); + TNode<IntPtrT> start_address = + IntPtrAdd(BitcastTaggedToWord(object), + IntPtrConstant(start_offset - kHeapObjectTag)); + TNode<IntPtrT> end_address = IntPtrAdd(start_address, size); StoreFieldsNoWriteBarrier(start_address, end_address, filler); } @@ -3883,8 +3839,9 @@ TNode<JSObject> CodeStubAssembler::AllocateJSObjectFromMap( } void CodeStubAssembler::InitializeJSObjectFromMap( - Node* object, Node* map, Node* instance_size, Node* properties, - Node* elements, SlackTrackingMode slack_tracking_mode) { + SloppyTNode<HeapObject> object, SloppyTNode<Map> map, + SloppyTNode<IntPtrT> instance_size, Node* properties, Node* elements, + SlackTrackingMode slack_tracking_mode) { CSA_SLOW_ASSERT(this, IsMap(map)); // This helper assumes that the object is in new-space, as guarded by the // check in AllocatedJSObjectFromMap. @@ -3915,7 +3872,8 @@ void CodeStubAssembler::InitializeJSObjectFromMap( } void CodeStubAssembler::InitializeJSObjectBodyNoSlackTracking( - Node* object, Node* map, Node* instance_size, int start_offset) { + SloppyTNode<HeapObject> object, SloppyTNode<Map> map, + SloppyTNode<IntPtrT> instance_size, int start_offset) { STATIC_ASSERT(Map::kNoSlackTracking == 0); CSA_ASSERT( this, IsClearWord32<Map::ConstructionCounterBits>(LoadMapBitField3(map))); @@ -3924,8 +3882,8 @@ void CodeStubAssembler::InitializeJSObjectBodyNoSlackTracking( } void CodeStubAssembler::InitializeJSObjectBodyWithSlackTracking( - Node* object, Node* map, Node* instance_size) { - CSA_SLOW_ASSERT(this, IsMap(map)); + SloppyTNode<HeapObject> object, SloppyTNode<Map> map, + SloppyTNode<IntPtrT> instance_size) { Comment("InitializeJSObjectBodyNoSlackTracking"); // Perform in-object slack tracking if requested. @@ -3953,9 +3911,9 @@ void CodeStubAssembler::InitializeJSObjectBodyWithSlackTracking( // The object still has in-object slack therefore the |unsed_or_unused| // field contain the "used" value. - TNode<UintPtrT> used_size = TimesTaggedSize(ChangeUint32ToWord( + TNode<IntPtrT> used_size = Signed(TimesTaggedSize(ChangeUint32ToWord( LoadObjectField(map, Map::kUsedOrUnusedInstanceSizeInWordsOffset, - MachineType::Uint8()))); + MachineType::Uint8())))); Comment("iInitialize filler fields"); InitializeFieldsWithRoot(object, used_size, instance_size, @@ -3984,19 +3942,19 @@ void CodeStubAssembler::InitializeJSObjectBodyWithSlackTracking( BIND(&end); } -void CodeStubAssembler::StoreFieldsNoWriteBarrier(Node* start_address, - Node* end_address, - Node* value) { +void CodeStubAssembler::StoreFieldsNoWriteBarrier(TNode<IntPtrT> start_address, + TNode<IntPtrT> end_address, + TNode<Object> value) { Comment("StoreFieldsNoWriteBarrier"); CSA_ASSERT(this, WordIsAligned(start_address, kTaggedSize)); CSA_ASSERT(this, WordIsAligned(end_address, kTaggedSize)); - BuildFastLoop( + BuildFastLoop<IntPtrT>( start_address, end_address, - [this, value](Node* current) { + [=](TNode<IntPtrT> current) { UnsafeStoreNoWriteBarrier(MachineRepresentation::kTagged, current, value); }, - kTaggedSize, INTPTR_PARAMETERS, IndexAdvanceMode::kPost); + kTaggedSize, IndexAdvanceMode::kPost); } TNode<BoolT> CodeStubAssembler::IsValidFastJSArrayCapacity( @@ -4008,12 +3966,12 @@ TNode<BoolT> CodeStubAssembler::IsValidFastJSArrayCapacity( TNode<JSArray> CodeStubAssembler::AllocateJSArray( TNode<Map> array_map, TNode<FixedArrayBase> elements, TNode<Smi> length, - Node* allocation_site, int array_header_size) { + TNode<AllocationSite> allocation_site, int array_header_size) { Comment("begin allocation of JSArray passing in elements"); CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length)); int base_size = array_header_size; - if (allocation_site != nullptr) { + if (!allocation_site.is_null()) { base_size += AllocationMemento::kSize; } @@ -4027,8 +3985,9 @@ TNode<JSArray> CodeStubAssembler::AllocateJSArray( std::pair<TNode<JSArray>, TNode<FixedArrayBase>> CodeStubAssembler::AllocateUninitializedJSArrayWithElements( ElementsKind kind, TNode<Map> array_map, TNode<Smi> length, - Node* allocation_site, Node* capacity, ParameterMode capacity_mode, - AllocationFlags allocation_flags, int array_header_size) { + TNode<AllocationSite> allocation_site, Node* capacity, + ParameterMode capacity_mode, AllocationFlags allocation_flags, + int array_header_size) { Comment("begin allocation of JSArray with elements"); CHECK_EQ(allocation_flags & ~kAllowLargeObjectAllocation, 0); CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length)); @@ -4065,7 +4024,9 @@ CodeStubAssembler::AllocateUninitializedJSArrayWithElements( BIND(&nonempty); { int base_size = array_header_size; - if (allocation_site != nullptr) base_size += AllocationMemento::kSize; + if (!allocation_site.is_null()) { + base_size += AllocationMemento::kSize; + } const int elements_offset = base_size; @@ -4138,8 +4099,8 @@ CodeStubAssembler::AllocateUninitializedJSArrayWithElements( } TNode<JSArray> CodeStubAssembler::AllocateUninitializedJSArray( - TNode<Map> array_map, TNode<Smi> length, Node* allocation_site, - TNode<IntPtrT> size_in_bytes) { + TNode<Map> array_map, TNode<Smi> length, + TNode<AllocationSite> allocation_site, TNode<IntPtrT> size_in_bytes) { CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length)); // Allocate space for the JSArray and the elements FixedArray in one go. @@ -4150,7 +4111,7 @@ TNode<JSArray> CodeStubAssembler::AllocateUninitializedJSArray( StoreObjectFieldRoot(array, JSArray::kPropertiesOrHashOffset, RootIndex::kEmptyFixedArray); - if (allocation_site != nullptr) { + if (!allocation_site.is_null()) { InitializeAllocationMemento(array, IntPtrConstant(JSArray::kSize), allocation_site); } @@ -4160,7 +4121,7 @@ TNode<JSArray> CodeStubAssembler::AllocateUninitializedJSArray( TNode<JSArray> CodeStubAssembler::AllocateJSArray( ElementsKind kind, TNode<Map> array_map, Node* capacity, TNode<Smi> length, - Node* allocation_site, ParameterMode capacity_mode, + TNode<AllocationSite> allocation_site, ParameterMode capacity_mode, AllocationFlags allocation_flags) { CSA_SLOW_ASSERT(this, TaggedIsPositiveSmi(length)); CSA_SLOW_ASSERT(this, MatchesParameterMode(capacity, capacity_mode)); @@ -4189,10 +4150,9 @@ TNode<JSArray> CodeStubAssembler::AllocateJSArray( return array; } -Node* CodeStubAssembler::ExtractFastJSArray(Node* context, Node* array, - Node* begin, Node* count, - ParameterMode mode, Node* capacity, - Node* allocation_site) { +Node* CodeStubAssembler::ExtractFastJSArray( + TNode<Context> context, TNode<JSArray> array, Node* begin, Node* count, + ParameterMode mode, Node* capacity, TNode<AllocationSite> allocation_site) { TNode<Map> original_array_map = LoadMap(array); TNode<Int32T> elements_kind = LoadMapElementsKind(original_array_map); @@ -4209,18 +4169,16 @@ Node* CodeStubAssembler::ExtractFastJSArray(Node* context, Node* array, return result; } -Node* CodeStubAssembler::CloneFastJSArray(Node* context, Node* array, - ParameterMode mode, - Node* allocation_site, - HoleConversionMode convert_holes) { +TNode<JSArray> CodeStubAssembler::CloneFastJSArray( + TNode<Context> context, TNode<JSArray> array, ParameterMode mode, + TNode<AllocationSite> allocation_site, HoleConversionMode convert_holes) { // TODO(dhai): we should be able to assert IsFastJSArray(array) here, but this // function is also used to copy boilerplates even when the no-elements // protector is invalid. This function should be renamed to reflect its uses. - CSA_ASSERT(this, IsJSArray(array)); TNode<Number> length = LoadJSArrayLength(array); - Node* new_elements = nullptr; - VARIABLE(var_new_elements, MachineRepresentation::kTagged); + TNode<FixedArrayBase> new_elements; + TVARIABLE(FixedArrayBase, var_new_elements); TVARIABLE(Int32T, var_elements_kind, LoadMapElementsKind(LoadMap(array))); Label allocate_jsarray(this), holey_extract(this), @@ -4240,7 +4198,7 @@ Node* CodeStubAssembler::CloneFastJSArray(Node* context, Node* array, TaggedToParameter(CAST(length), mode), nullptr, ExtractFixedArrayFlag::kAllFixedArraysDontCopyCOW, mode, nullptr, var_elements_kind.value()); - var_new_elements.Bind(new_elements); + var_new_elements = new_elements; Goto(&allocate_jsarray); if (need_conversion) { @@ -4257,7 +4215,7 @@ Node* CodeStubAssembler::CloneFastJSArray(Node* context, Node* array, LoadElements(array), IntPtrOrSmiConstant(0, mode), TaggedToParameter(CAST(length), mode), nullptr, ExtractFixedArrayFlag::kAllFixedArrays, mode, &var_holes_converted); - var_new_elements.Bind(new_elements); + var_new_elements = new_elements; // If the array type didn't change, use the original elements kind. GotoIfNot(var_holes_converted.value(), &allocate_jsarray); // Otherwise use PACKED_ELEMENTS for the target's elements kind. @@ -4283,8 +4241,8 @@ Node* CodeStubAssembler::CloneFastJSArray(Node* context, Node* array, TNode<Map> array_map = LoadJSArrayElementsMap(var_elements_kind.value(), native_context); - TNode<JSArray> result = AllocateJSArray( - array_map, CAST(var_new_elements.value()), CAST(length), allocation_site); + TNode<JSArray> result = AllocateJSArray(array_map, var_new_elements.value(), + CAST(length), allocation_site); return result; } @@ -4555,14 +4513,14 @@ TNode<FixedArrayBase> CodeStubAssembler::ExtractFixedDoubleArrayFillingHoles( const int first_element_offset = FixedArray::kHeaderSize - kHeapObjectTag; TNode<IntPtrT> first_from_element_offset = ElementOffsetFromIndex(first, kind, mode, 0); - TNode<WordT> limit_offset = IntPtrAdd(first_from_element_offset, - IntPtrConstant(first_element_offset)); + TNode<IntPtrT> limit_offset = IntPtrAdd(first_from_element_offset, + IntPtrConstant(first_element_offset)); TVARIABLE(IntPtrT, var_from_offset, ElementOffsetFromIndex(IntPtrOrSmiAdd(first, count, mode), kind, mode, first_element_offset)); Label decrement(this, {&var_from_offset}), done(this); - TNode<WordT> to_array_adjusted = + TNode<IntPtrT> to_array_adjusted = IntPtrSub(BitcastTaggedToWord(to_elements), first_from_element_offset); Branch(WordEqual(var_from_offset.value(), limit_offset), &done, &decrement); @@ -4908,12 +4866,10 @@ void CodeStubAssembler::MoveElements(ElementsKind kind, TNode<IntPtrT> elements_intptr = BitcastTaggedToWord(elements); TNode<IntPtrT> target_data_ptr = IntPtrAdd(elements_intptr, - ElementOffsetFromIndex(dst_index, kind, INTPTR_PARAMETERS, - fa_base_data_offset)); + ElementOffsetFromIndex(dst_index, kind, fa_base_data_offset)); TNode<IntPtrT> source_data_ptr = IntPtrAdd(elements_intptr, - ElementOffsetFromIndex(src_index, kind, INTPTR_PARAMETERS, - fa_base_data_offset)); + ElementOffsetFromIndex(src_index, kind, fa_base_data_offset)); TNode<ExternalReference> memmove = ExternalConstant(ExternalReference::libc_memmove_function()); CallCFunction(memmove, MachineType::Pointer(), @@ -4997,10 +4953,10 @@ void CodeStubAssembler::CopyElements(ElementsKind kind, IntPtrMul(length, IntPtrConstant(ElementsKindToByteSize(kind))); static const int32_t fa_base_data_offset = FixedArrayBase::kHeaderSize - kHeapObjectTag; - TNode<IntPtrT> src_offset_start = ElementOffsetFromIndex( - src_index, kind, INTPTR_PARAMETERS, fa_base_data_offset); - TNode<IntPtrT> dst_offset_start = ElementOffsetFromIndex( - dst_index, kind, INTPTR_PARAMETERS, fa_base_data_offset); + TNode<IntPtrT> src_offset_start = + ElementOffsetFromIndex(src_index, kind, fa_base_data_offset); + TNode<IntPtrT> dst_offset_start = + ElementOffsetFromIndex(dst_index, kind, fa_base_data_offset); TNode<IntPtrT> src_elements_intptr = BitcastTaggedToWord(src_elements); TNode<IntPtrT> source_data_ptr = IntPtrAdd(src_elements_intptr, src_offset_start); @@ -5283,65 +5239,6 @@ void CodeStubAssembler::CopyPropertyArrayValues(Node* from_array, Comment("] CopyPropertyArrayValues"); } -void CodeStubAssembler::CopyStringCharacters(Node* from_string, Node* to_string, - TNode<IntPtrT> from_index, - TNode<IntPtrT> to_index, - TNode<IntPtrT> character_count, - String::Encoding from_encoding, - String::Encoding to_encoding) { - // Cannot assert IsString(from_string) and IsString(to_string) here because - // CSA::SubString can pass in faked sequential strings when handling external - // subject strings. - bool from_one_byte = from_encoding == String::ONE_BYTE_ENCODING; - bool to_one_byte = to_encoding == String::ONE_BYTE_ENCODING; - DCHECK_IMPLIES(to_one_byte, from_one_byte); - Comment("CopyStringCharacters ", - from_one_byte ? "ONE_BYTE_ENCODING" : "TWO_BYTE_ENCODING", " -> ", - to_one_byte ? "ONE_BYTE_ENCODING" : "TWO_BYTE_ENCODING"); - - ElementsKind from_kind = from_one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS; - ElementsKind to_kind = to_one_byte ? UINT8_ELEMENTS : UINT16_ELEMENTS; - STATIC_ASSERT(SeqOneByteString::kHeaderSize == SeqTwoByteString::kHeaderSize); - int header_size = SeqOneByteString::kHeaderSize - kHeapObjectTag; - TNode<IntPtrT> from_offset = ElementOffsetFromIndex( - from_index, from_kind, INTPTR_PARAMETERS, header_size); - TNode<IntPtrT> to_offset = - ElementOffsetFromIndex(to_index, to_kind, INTPTR_PARAMETERS, header_size); - TNode<IntPtrT> byte_count = - ElementOffsetFromIndex(character_count, from_kind, INTPTR_PARAMETERS); - TNode<WordT> limit_offset = IntPtrAdd(from_offset, byte_count); - - // Prepare the fast loop - MachineType type = - from_one_byte ? MachineType::Uint8() : MachineType::Uint16(); - MachineRepresentation rep = to_one_byte ? MachineRepresentation::kWord8 - : MachineRepresentation::kWord16; - int from_increment = 1 << ElementsKindToShiftSize(from_kind); - int to_increment = 1 << ElementsKindToShiftSize(to_kind); - - VARIABLE(current_to_offset, MachineType::PointerRepresentation(), to_offset); - VariableList vars({¤t_to_offset}, zone()); - int to_index_constant = 0, from_index_constant = 0; - bool index_same = (from_encoding == to_encoding) && - (from_index == to_index || - (ToInt32Constant(from_index, &from_index_constant) && - ToInt32Constant(to_index, &to_index_constant) && - from_index_constant == to_index_constant)); - BuildFastLoop( - vars, from_offset, limit_offset, - [this, from_string, to_string, ¤t_to_offset, to_increment, type, - rep, index_same](Node* offset) { - Node* value = Load(type, from_string, offset); - StoreNoWriteBarrier(rep, to_string, - index_same ? offset : current_to_offset.value(), - value); - if (!index_same) { - Increment(¤t_to_offset, to_increment); - } - }, - from_increment, INTPTR_PARAMETERS, IndexAdvanceMode::kPost); -} - Node* CodeStubAssembler::LoadElementAndPrepareForStore(Node* array, Node* offset, ElementsKind from_kind, @@ -5381,9 +5278,9 @@ Node* CodeStubAssembler::CalculateNewElementsCapacity(Node* old_capacity, return IntPtrOrSmiAdd(new_capacity, padding, mode); } -Node* CodeStubAssembler::TryGrowElementsCapacity(Node* object, Node* elements, - ElementsKind kind, Node* key, - Label* bailout) { +TNode<FixedArrayBase> CodeStubAssembler::TryGrowElementsCapacity( + Node* object, Node* elements, ElementsKind kind, Node* key, + Label* bailout) { CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object)); CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, kind)); CSA_SLOW_ASSERT(this, TaggedIsSmi(key)); @@ -5395,11 +5292,9 @@ Node* CodeStubAssembler::TryGrowElementsCapacity(Node* object, Node* elements, TaggedToParameter(capacity, mode), mode, bailout); } -Node* CodeStubAssembler::TryGrowElementsCapacity(Node* object, Node* elements, - ElementsKind kind, Node* key, - Node* capacity, - ParameterMode mode, - Label* bailout) { +TNode<FixedArrayBase> CodeStubAssembler::TryGrowElementsCapacity( + Node* object, Node* elements, ElementsKind kind, Node* key, Node* capacity, + ParameterMode mode, Label* bailout) { Comment("TryGrowElementsCapacity"); CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object)); CSA_SLOW_ASSERT(this, IsFixedArrayWithKindOrEmpty(elements, kind)); @@ -5418,7 +5313,7 @@ Node* CodeStubAssembler::TryGrowElementsCapacity(Node* object, Node* elements, new_capacity, mode, bailout); } -Node* CodeStubAssembler::GrowElementsCapacity( +TNode<FixedArrayBase> CodeStubAssembler::GrowElementsCapacity( Node* object, Node* elements, ElementsKind from_kind, ElementsKind to_kind, Node* capacity, Node* new_capacity, ParameterMode mode, Label* bailout) { Comment("[ GrowElementsCapacity"); @@ -5471,45 +5366,22 @@ void CodeStubAssembler::InitializeAllocationMemento(Node* base, Comment("]"); } -Node* CodeStubAssembler::TryTaggedToFloat64(Node* value, - Label* if_valueisnotnumber) { - Label out(this); - VARIABLE(var_result, MachineRepresentation::kFloat64); - - // Check if the {value} is a Smi or a HeapObject. - Label if_valueissmi(this), if_valueisnotsmi(this); - Branch(TaggedIsSmi(value), &if_valueissmi, &if_valueisnotsmi); - - BIND(&if_valueissmi); - { - // Convert the Smi {value}. - var_result.Bind(SmiToFloat64(value)); - Goto(&out); - } - - BIND(&if_valueisnotsmi); - { - // Check if {value} is a HeapNumber. - Label if_valueisheapnumber(this); - Branch(IsHeapNumber(value), &if_valueisheapnumber, if_valueisnotnumber); - - BIND(&if_valueisheapnumber); - { - // Load the floating point value. - var_result.Bind(LoadHeapNumberValue(value)); - Goto(&out); - } - } - BIND(&out); - return var_result.value(); +TNode<Float64T> CodeStubAssembler::TryTaggedToFloat64( + TNode<Object> value, Label* if_valueisnotnumber) { + return Select<Float64T>( + TaggedIsSmi(value), [&]() { return SmiToFloat64(CAST(value)); }, + [&]() { + GotoIfNot(IsHeapNumber(CAST(value)), if_valueisnotnumber); + return LoadHeapNumberValue(CAST(value)); + }); } -Node* CodeStubAssembler::TruncateTaggedToFloat64(Node* context, Node* value) { +TNode<Float64T> CodeStubAssembler::TruncateTaggedToFloat64( + SloppyTNode<Context> context, SloppyTNode<Object> value) { // We might need to loop once due to ToNumber conversion. - VARIABLE(var_value, MachineRepresentation::kTagged); - VARIABLE(var_result, MachineRepresentation::kFloat64); + TVARIABLE(Object, var_value, value); + TVARIABLE(Float64T, var_result); Label loop(this, &var_value), done_loop(this, &var_result); - var_value.Bind(value); Goto(&loop); BIND(&loop); { @@ -5520,14 +5392,13 @@ Node* CodeStubAssembler::TruncateTaggedToFloat64(Node* context, Node* value) { // Convert {value} to Float64 if it is a number and convert it to a number // otherwise. - Node* const result = TryTaggedToFloat64(value, &if_valueisnotnumber); - var_result.Bind(result); + var_result = TryTaggedToFloat64(value, &if_valueisnotnumber); Goto(&done_loop); BIND(&if_valueisnotnumber); { // Convert the {value} to a Number first. - var_value.Bind(CallBuiltin(Builtins::kNonNumberToNumber, context, value)); + var_value = CallBuiltin(Builtins::kNonNumberToNumber, context, value); Goto(&loop); } } @@ -5535,8 +5406,9 @@ Node* CodeStubAssembler::TruncateTaggedToFloat64(Node* context, Node* value) { return var_result.value(); } -Node* CodeStubAssembler::TruncateTaggedToWord32(Node* context, Node* value) { - VARIABLE(var_result, MachineRepresentation::kWord32); +TNode<Word32T> CodeStubAssembler::TruncateTaggedToWord32( + SloppyTNode<Context> context, SloppyTNode<Object> value) { + TVARIABLE(Word32T, var_result); Label done(this); TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumber>(context, value, &done, &var_result); @@ -5546,38 +5418,33 @@ Node* CodeStubAssembler::TruncateTaggedToWord32(Node* context, Node* value) { // Truncate {value} to word32 and jump to {if_number} if it is a Number, // or find that it is a BigInt and jump to {if_bigint}. -void CodeStubAssembler::TaggedToWord32OrBigInt(Node* context, Node* value, - Label* if_number, - Variable* var_word32, - Label* if_bigint, - Variable* var_bigint) { +void CodeStubAssembler::TaggedToWord32OrBigInt( + TNode<Context> context, TNode<Object> value, Label* if_number, + TVariable<Word32T>* var_word32, Label* if_bigint, + TVariable<Object>* var_maybe_bigint) { TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumeric>( - context, value, if_number, var_word32, if_bigint, var_bigint); + context, value, if_number, var_word32, if_bigint, var_maybe_bigint); } // Truncate {value} to word32 and jump to {if_number} if it is a Number, // or find that it is a BigInt and jump to {if_bigint}. In either case, // store the type feedback in {var_feedback}. void CodeStubAssembler::TaggedToWord32OrBigIntWithFeedback( - Node* context, Node* value, Label* if_number, Variable* var_word32, - Label* if_bigint, Variable* var_bigint, Variable* var_feedback) { + TNode<Context> context, TNode<Object> value, Label* if_number, + TVariable<Word32T>* var_word32, Label* if_bigint, + TVariable<Object>* var_maybe_bigint, TVariable<Smi>* var_feedback) { TaggedToWord32OrBigIntImpl<Object::Conversion::kToNumeric>( - context, value, if_number, var_word32, if_bigint, var_bigint, + context, value, if_number, var_word32, if_bigint, var_maybe_bigint, var_feedback); } template <Object::Conversion conversion> void CodeStubAssembler::TaggedToWord32OrBigIntImpl( - Node* context, Node* value, Label* if_number, Variable* var_word32, - Label* if_bigint, Variable* var_bigint, Variable* var_feedback) { - DCHECK(var_word32->rep() == MachineRepresentation::kWord32); - DCHECK(var_bigint == nullptr || - var_bigint->rep() == MachineRepresentation::kTagged); - DCHECK(var_feedback == nullptr || - var_feedback->rep() == MachineRepresentation::kTaggedSigned); - + TNode<Context> context, TNode<Object> value, Label* if_number, + TVariable<Word32T>* var_word32, Label* if_bigint, + TVariable<Object>* var_maybe_bigint, TVariable<Smi>* var_feedback) { // We might need to loop after conversion. - VARIABLE(var_value, MachineRepresentation::kTagged, value); + TVARIABLE(Object, var_value, value); OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNone); Variable* loop_vars[] = {&var_value, var_feedback}; int num_vars = @@ -5592,12 +5459,13 @@ void CodeStubAssembler::TaggedToWord32OrBigIntImpl( GotoIf(TaggedIsNotSmi(value), ¬_smi); // {value} is a Smi. - var_word32->Bind(SmiToInt32(value)); + *var_word32 = SmiToInt32(CAST(value)); CombineFeedback(var_feedback, BinaryOperationFeedback::kSignedSmall); Goto(if_number); BIND(¬_smi); - TNode<Map> map = LoadMap(value); + TNode<HeapObject> value_heap_object = CAST(value); + TNode<Map> map = LoadMap(value_heap_object); GotoIf(IsHeapNumberMap(map), &is_heap_number); TNode<Uint16T> instance_type = LoadMapInstanceType(map); if (conversion == Object::Conversion::kToNumeric) { @@ -5610,7 +5478,7 @@ void CodeStubAssembler::TaggedToWord32OrBigIntImpl( // We do not require an Or with earlier feedback here because once we // convert the value to a Numeric, we cannot reach this path. We can // only reach this path on the first pass when the feedback is kNone. - CSA_ASSERT(this, SmiEqual(CAST(var_feedback->value()), + CSA_ASSERT(this, SmiEqual(var_feedback->value(), SmiConstant(BinaryOperationFeedback::kNone))); } GotoIf(InstanceTypeEqual(instance_type, ODDBALL_TYPE), &is_oddball); @@ -5618,25 +5486,25 @@ void CodeStubAssembler::TaggedToWord32OrBigIntImpl( auto builtin = conversion == Object::Conversion::kToNumeric ? Builtins::kNonNumberToNumeric : Builtins::kNonNumberToNumber; - var_value.Bind(CallBuiltin(builtin, context, value)); + var_value = CallBuiltin(builtin, context, value); OverwriteFeedback(var_feedback, BinaryOperationFeedback::kAny); Goto(&loop); BIND(&is_oddball); - var_value.Bind(LoadObjectField(value, Oddball::kToNumberOffset)); + var_value = LoadObjectField(value_heap_object, Oddball::kToNumberOffset); OverwriteFeedback(var_feedback, BinaryOperationFeedback::kNumberOrOddball); Goto(&loop); } BIND(&is_heap_number); - var_word32->Bind(TruncateHeapNumberValueToWord32(CAST(value))); + *var_word32 = TruncateHeapNumberValueToWord32(CAST(value)); CombineFeedback(var_feedback, BinaryOperationFeedback::kNumber); Goto(if_number); if (conversion == Object::Conversion::kToNumeric) { BIND(&is_bigint); - var_bigint->Bind(value); + *var_maybe_bigint = value; CombineFeedback(var_feedback, BinaryOperationFeedback::kBigInt); Goto(if_bigint); } @@ -5650,14 +5518,14 @@ TNode<Int32T> CodeStubAssembler::TruncateHeapNumberValueToWord32( } void CodeStubAssembler::TryHeapNumberToSmi(TNode<HeapNumber> number, - TVariable<Smi>& var_result_smi, + TVariable<Smi>* var_result_smi, Label* if_smi) { TNode<Float64T> value = LoadHeapNumberValue(number); TryFloat64ToSmi(value, var_result_smi, if_smi); } void CodeStubAssembler::TryFloat64ToSmi(TNode<Float64T> value, - TVariable<Smi>& var_result_smi, + TVariable<Smi>* var_result_smi, Label* if_smi) { TNode<Int32T> value32 = RoundFloat64ToInt32(value); TNode<Float64T> value64 = ChangeInt32ToFloat64(value32); @@ -5674,13 +5542,13 @@ void CodeStubAssembler::TryFloat64ToSmi(TNode<Float64T> value, BIND(&if_int32); { if (SmiValuesAre32Bits()) { - var_result_smi = SmiTag(ChangeInt32ToIntPtr(value32)); + *var_result_smi = SmiTag(ChangeInt32ToIntPtr(value32)); } else { DCHECK(SmiValuesAre31Bits()); TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow(value32, value32); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, &if_heap_number); - var_result_smi = + *var_result_smi = BitcastWordToTaggedSigned(ChangeInt32ToIntPtr(Projection<0>(pair))); } Goto(if_smi); @@ -5693,7 +5561,7 @@ TNode<Number> CodeStubAssembler::ChangeFloat64ToTagged( Label if_smi(this), done(this); TVARIABLE(Smi, var_smi_result); TVARIABLE(Number, var_result); - TryFloat64ToSmi(value, var_smi_result, &if_smi); + TryFloat64ToSmi(value, &var_smi_result, &if_smi); var_result = AllocateHeapNumberWithValue(value); Goto(&done); @@ -6144,42 +6012,42 @@ TNode<BoolT> CodeStubAssembler::IsUndetectableMap(SloppyTNode<Map> map) { } TNode<BoolT> CodeStubAssembler::IsNoElementsProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); TNode<PropertyCell> cell = NoElementsProtectorConstant(); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); } TNode<BoolT> CodeStubAssembler::IsArrayIteratorProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); TNode<PropertyCell> cell = ArrayIteratorProtectorConstant(); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); } TNode<BoolT> CodeStubAssembler::IsPromiseResolveProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); - TNode<Cell> cell = PromiseResolveProtectorConstant(); - TNode<Object> cell_value = LoadObjectField(cell, Cell::kValueOffset); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); + TNode<PropertyCell> cell = PromiseResolveProtectorConstant(); + TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); } TNode<BoolT> CodeStubAssembler::IsPromiseThenProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); TNode<PropertyCell> cell = PromiseThenProtectorConstant(); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); } TNode<BoolT> CodeStubAssembler::IsArraySpeciesProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); TNode<PropertyCell> cell = ArraySpeciesProtectorConstant(); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); } TNode<BoolT> CodeStubAssembler::IsTypedArraySpeciesProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); TNode<PropertyCell> cell = TypedArraySpeciesProtectorConstant(); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); @@ -6190,12 +6058,12 @@ TNode<BoolT> CodeStubAssembler::IsRegExpSpeciesProtectorCellInvalid( TNode<PropertyCell> cell = CAST(LoadContextElement( native_context, Context::REGEXP_SPECIES_PROTECTOR_INDEX)); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); return TaggedEqual(cell_value, invalid); } TNode<BoolT> CodeStubAssembler::IsPromiseSpeciesProtectorCellInvalid() { - TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); + TNode<Smi> invalid = SmiConstant(Protectors::kProtectorInvalid); TNode<PropertyCell> cell = PromiseSpeciesProtectorConstant(); TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); return TaggedEqual(cell_value, invalid); @@ -6394,6 +6262,10 @@ TNode<BoolT> CodeStubAssembler::IsJSGlobalProxy( return IsJSGlobalProxyMap(LoadMap(object)); } +TNode<BoolT> CodeStubAssembler::IsJSGeneratorMap(TNode<Map> map) { + return InstanceTypeEqual(LoadMapInstanceType(map), JS_GENERATOR_OBJECT_TYPE); +} + TNode<BoolT> CodeStubAssembler::IsJSObjectInstanceType( SloppyTNode<Int32T> instance_type) { STATIC_ASSERT(LAST_JS_OBJECT_TYPE == LAST_TYPE); @@ -6428,6 +6300,11 @@ TNode<BoolT> CodeStubAssembler::IsJSStringIterator( return HasInstanceType(object, JS_STRING_ITERATOR_TYPE); } +TNode<BoolT> CodeStubAssembler::IsJSRegExpStringIterator( + SloppyTNode<HeapObject> object) { + return HasInstanceType(object, JS_REG_EXP_STRING_ITERATOR_TYPE); +} + TNode<BoolT> CodeStubAssembler::IsMap(SloppyTNode<HeapObject> map) { return IsMetaMap(LoadMap(map)); } @@ -6656,7 +6533,7 @@ TNode<BoolT> CodeStubAssembler::IsBigInt(SloppyTNode<HeapObject> object) { TNode<BoolT> CodeStubAssembler::IsPrimitiveInstanceType( SloppyTNode<Int32T> instance_type) { return Int32LessThanOrEqual(instance_type, - Int32Constant(LAST_PRIMITIVE_TYPE)); + Int32Constant(LAST_PRIMITIVE_HEAP_OBJECT_TYPE)); } TNode<BoolT> CodeStubAssembler::IsPrivateSymbol( @@ -6716,8 +6593,7 @@ TNode<BoolT> CodeStubAssembler::IsNumberDictionary( return HasInstanceType(object, NUMBER_DICTIONARY_TYPE); } -TNode<BoolT> CodeStubAssembler::IsJSGeneratorObject( - SloppyTNode<HeapObject> object) { +TNode<BoolT> CodeStubAssembler::IsJSGeneratorObject(TNode<HeapObject> object) { return HasInstanceType(object, JS_GENERATOR_OBJECT_TYPE); } @@ -6762,7 +6638,7 @@ TNode<BoolT> CodeStubAssembler::IsJSDataView(TNode<HeapObject> object) { } TNode<BoolT> CodeStubAssembler::IsJSRegExp(SloppyTNode<HeapObject> object) { - return HasInstanceType(object, JS_REGEXP_TYPE); + return HasInstanceType(object, JS_REG_EXP_TYPE); } TNode<BoolT> CodeStubAssembler::IsNumber(SloppyTNode<Object> object) { @@ -7011,189 +6887,6 @@ TNode<String> CodeStubAssembler::StringFromSingleCharCode(TNode<Int32T> code) { return CAST(var_result.value()); } -// A wrapper around CopyStringCharacters which determines the correct string -// encoding, allocates a corresponding sequential string, and then copies the -// given character range using CopyStringCharacters. -// |from_string| must be a sequential string. -// 0 <= |from_index| <= |from_index| + |character_count| < from_string.length. -TNode<String> CodeStubAssembler::AllocAndCopyStringCharacters( - Node* from, Node* from_instance_type, TNode<IntPtrT> from_index, - TNode<IntPtrT> character_count) { - Label end(this), one_byte_sequential(this), two_byte_sequential(this); - TVARIABLE(String, var_result); - - Branch(IsOneByteStringInstanceType(from_instance_type), &one_byte_sequential, - &two_byte_sequential); - - // The subject string is a sequential one-byte string. - BIND(&one_byte_sequential); - { - TNode<String> result = AllocateSeqOneByteString( - Unsigned(TruncateIntPtrToInt32(character_count))); - CopyStringCharacters(from, result, from_index, IntPtrConstant(0), - character_count, String::ONE_BYTE_ENCODING, - String::ONE_BYTE_ENCODING); - var_result = result; - Goto(&end); - } - - // The subject string is a sequential two-byte string. - BIND(&two_byte_sequential); - { - TNode<String> result = AllocateSeqTwoByteString( - Unsigned(TruncateIntPtrToInt32(character_count))); - CopyStringCharacters(from, result, from_index, IntPtrConstant(0), - character_count, String::TWO_BYTE_ENCODING, - String::TWO_BYTE_ENCODING); - var_result = result; - Goto(&end); - } - - BIND(&end); - return var_result.value(); -} - -TNode<String> CodeStubAssembler::SubString(TNode<String> string, - TNode<IntPtrT> from, - TNode<IntPtrT> to) { - TVARIABLE(String, var_result); - ToDirectStringAssembler to_direct(state(), string); - Label end(this), runtime(this); - - TNode<IntPtrT> const substr_length = IntPtrSub(to, from); - TNode<IntPtrT> const string_length = LoadStringLengthAsWord(string); - - // Begin dispatching based on substring length. - - Label original_string_or_invalid_length(this); - GotoIf(UintPtrGreaterThanOrEqual(substr_length, string_length), - &original_string_or_invalid_length); - - // A real substring (substr_length < string_length). - Label empty(this); - GotoIf(IntPtrEqual(substr_length, IntPtrConstant(0)), &empty); - - Label single_char(this); - GotoIf(IntPtrEqual(substr_length, IntPtrConstant(1)), &single_char); - - // Deal with different string types: update the index if necessary - // and extract the underlying string. - - TNode<String> direct_string = to_direct.TryToDirect(&runtime); - TNode<IntPtrT> offset = IntPtrAdd(from, to_direct.offset()); - TNode<Int32T> const instance_type = to_direct.instance_type(); - - // The subject string can only be external or sequential string of either - // encoding at this point. - Label external_string(this); - { - if (FLAG_string_slices) { - Label next(this); - - // Short slice. Copy instead of slicing. - GotoIf(IntPtrLessThan(substr_length, - IntPtrConstant(SlicedString::kMinLength)), - &next); - - // Allocate new sliced string. - - Counters* counters = isolate()->counters(); - IncrementCounter(counters->sub_string_native(), 1); - - Label one_byte_slice(this), two_byte_slice(this); - Branch(IsOneByteStringInstanceType(to_direct.instance_type()), - &one_byte_slice, &two_byte_slice); - - BIND(&one_byte_slice); - { - var_result = AllocateSlicedOneByteString( - Unsigned(TruncateIntPtrToInt32(substr_length)), direct_string, - SmiTag(offset)); - Goto(&end); - } - - BIND(&two_byte_slice); - { - var_result = AllocateSlicedTwoByteString( - Unsigned(TruncateIntPtrToInt32(substr_length)), direct_string, - SmiTag(offset)); - Goto(&end); - } - - BIND(&next); - } - - // The subject string can only be external or sequential string of either - // encoding at this point. - GotoIf(to_direct.is_external(), &external_string); - - var_result = AllocAndCopyStringCharacters(direct_string, instance_type, - offset, substr_length); - - Counters* counters = isolate()->counters(); - IncrementCounter(counters->sub_string_native(), 1); - - Goto(&end); - } - - // Handle external string. - BIND(&external_string); - { - TNode<RawPtrT> const fake_sequential_string = - to_direct.PointerToString(&runtime); - - var_result = AllocAndCopyStringCharacters( - fake_sequential_string, instance_type, offset, substr_length); - - Counters* counters = isolate()->counters(); - IncrementCounter(counters->sub_string_native(), 1); - - Goto(&end); - } - - BIND(&empty); - { - var_result = EmptyStringConstant(); - Goto(&end); - } - - // Substrings of length 1 are generated through CharCodeAt and FromCharCode. - BIND(&single_char); - { - TNode<Int32T> char_code = StringCharCodeAt(string, from); - var_result = StringFromSingleCharCode(char_code); - Goto(&end); - } - - BIND(&original_string_or_invalid_length); - { - CSA_ASSERT(this, IntPtrEqual(substr_length, string_length)); - - // Equal length - check if {from, to} == {0, str.length}. - GotoIf(UintPtrGreaterThan(from, IntPtrConstant(0)), &runtime); - - // Return the original string (substr_length == string_length). - - Counters* counters = isolate()->counters(); - IncrementCounter(counters->sub_string_native(), 1); - - var_result = string; - Goto(&end); - } - - // Fall back to a runtime call. - BIND(&runtime); - { - var_result = - CAST(CallRuntime(Runtime::kStringSubstring, NoContextConstant(), string, - SmiTag(from), SmiTag(to))); - Goto(&end); - } - - BIND(&end); - return var_result.value(); -} - ToDirectStringAssembler::ToDirectStringAssembler( compiler::CodeAssemblerState* state, TNode<String> string, Flags flags) : CodeStubAssembler(state), @@ -7204,8 +6897,7 @@ ToDirectStringAssembler::ToDirectStringAssembler( flags_(flags) {} TNode<String> ToDirectStringAssembler::TryToDirect(Label* if_bailout) { - VariableList vars({&var_string_, &var_offset_, &var_instance_type_}, zone()); - Label dispatch(this, vars); + Label dispatch(this, {&var_string_, &var_offset_, &var_instance_type_}); Label if_iscons(this); Label if_isexternal(this); Label if_issliced(this); @@ -7333,232 +7025,6 @@ TNode<RawPtrT> ToDirectStringAssembler::TryToSequential( return var_result.value(); } -void CodeStubAssembler::BranchIfCanDerefIndirectString( - TNode<String> string, TNode<Int32T> instance_type, Label* can_deref, - Label* cannot_deref) { - TNode<Int32T> representation = - Word32And(instance_type, Int32Constant(kStringRepresentationMask)); - GotoIf(Word32Equal(representation, Int32Constant(kThinStringTag)), can_deref); - GotoIf(Word32NotEqual(representation, Int32Constant(kConsStringTag)), - cannot_deref); - // Cons string. - TNode<String> rhs = - LoadObjectField<String>(string, ConsString::kSecondOffset); - GotoIf(IsEmptyString(rhs), can_deref); - Goto(cannot_deref); -} - -TNode<String> CodeStubAssembler::DerefIndirectString( - TNode<String> string, TNode<Int32T> instance_type, Label* cannot_deref) { - Label deref(this); - BranchIfCanDerefIndirectString(string, instance_type, &deref, cannot_deref); - BIND(&deref); - STATIC_ASSERT(static_cast<int>(ThinString::kActualOffset) == - static_cast<int>(ConsString::kFirstOffset)); - return LoadObjectField<String>(string, ThinString::kActualOffset); -} - -void CodeStubAssembler::DerefIndirectString(TVariable<String>* var_string, - TNode<Int32T> instance_type) { -#ifdef DEBUG - Label can_deref(this), cannot_deref(this); - BranchIfCanDerefIndirectString(var_string->value(), instance_type, &can_deref, - &cannot_deref); - BIND(&cannot_deref); - DebugBreak(); // Should be able to dereference string. - Goto(&can_deref); - BIND(&can_deref); -#endif // DEBUG - - STATIC_ASSERT(static_cast<int>(ThinString::kActualOffset) == - static_cast<int>(ConsString::kFirstOffset)); - *var_string = - LoadObjectField<String>(var_string->value(), ThinString::kActualOffset); -} - -void CodeStubAssembler::MaybeDerefIndirectString(TVariable<String>* var_string, - TNode<Int32T> instance_type, - Label* did_deref, - Label* cannot_deref) { - Label deref(this); - BranchIfCanDerefIndirectString(var_string->value(), instance_type, &deref, - cannot_deref); - - BIND(&deref); - { - DerefIndirectString(var_string, instance_type); - Goto(did_deref); - } -} - -void CodeStubAssembler::MaybeDerefIndirectStrings( - TVariable<String>* var_left, TNode<Int32T> left_instance_type, - TVariable<String>* var_right, TNode<Int32T> right_instance_type, - Label* did_something) { - Label did_nothing_left(this), did_something_left(this), - didnt_do_anything(this); - MaybeDerefIndirectString(var_left, left_instance_type, &did_something_left, - &did_nothing_left); - - BIND(&did_something_left); - { - MaybeDerefIndirectString(var_right, right_instance_type, did_something, - did_something); - } - - BIND(&did_nothing_left); - { - MaybeDerefIndirectString(var_right, right_instance_type, did_something, - &didnt_do_anything); - } - - BIND(&didnt_do_anything); - // Fall through if neither string was an indirect string. -} - -TNode<String> CodeStubAssembler::StringAdd(Node* context, TNode<String> left, - TNode<String> right) { - TVARIABLE(String, result); - Label check_right(this), runtime(this, Label::kDeferred), cons(this), - done(this, &result), done_native(this, &result); - Counters* counters = isolate()->counters(); - - TNode<Uint32T> left_length = LoadStringLengthAsWord32(left); - GotoIfNot(Word32Equal(left_length, Uint32Constant(0)), &check_right); - result = right; - Goto(&done_native); - - BIND(&check_right); - TNode<Uint32T> right_length = LoadStringLengthAsWord32(right); - GotoIfNot(Word32Equal(right_length, Uint32Constant(0)), &cons); - result = left; - Goto(&done_native); - - BIND(&cons); - { - TNode<Uint32T> new_length = Uint32Add(left_length, right_length); - - // If new length is greater than String::kMaxLength, goto runtime to - // throw. Note: we also need to invalidate the string length protector, so - // can't just throw here directly. - GotoIf(Uint32GreaterThan(new_length, Uint32Constant(String::kMaxLength)), - &runtime); - - TVARIABLE(String, var_left, left); - TVARIABLE(String, var_right, right); - Variable* input_vars[2] = {&var_left, &var_right}; - Label non_cons(this, 2, input_vars); - Label slow(this, Label::kDeferred); - GotoIf(Uint32LessThan(new_length, Uint32Constant(ConsString::kMinLength)), - &non_cons); - - result = - AllocateConsString(new_length, var_left.value(), var_right.value()); - Goto(&done_native); - - BIND(&non_cons); - - Comment("Full string concatenate"); - TNode<Int32T> left_instance_type = LoadInstanceType(var_left.value()); - TNode<Int32T> right_instance_type = LoadInstanceType(var_right.value()); - // Compute intersection and difference of instance types. - - TNode<Int32T> ored_instance_types = - Word32Or(left_instance_type, right_instance_type); - TNode<Word32T> xored_instance_types = - Word32Xor(left_instance_type, right_instance_type); - - // Check if both strings have the same encoding and both are sequential. - GotoIf(IsSetWord32(xored_instance_types, kStringEncodingMask), &runtime); - GotoIf(IsSetWord32(ored_instance_types, kStringRepresentationMask), &slow); - - TNode<IntPtrT> word_left_length = Signed(ChangeUint32ToWord(left_length)); - TNode<IntPtrT> word_right_length = Signed(ChangeUint32ToWord(right_length)); - - Label two_byte(this); - GotoIf(Word32Equal(Word32And(ored_instance_types, - Int32Constant(kStringEncodingMask)), - Int32Constant(kTwoByteStringTag)), - &two_byte); - // One-byte sequential string case - result = AllocateSeqOneByteString(new_length); - CopyStringCharacters(var_left.value(), result.value(), IntPtrConstant(0), - IntPtrConstant(0), word_left_length, - String::ONE_BYTE_ENCODING, String::ONE_BYTE_ENCODING); - CopyStringCharacters(var_right.value(), result.value(), IntPtrConstant(0), - word_left_length, word_right_length, - String::ONE_BYTE_ENCODING, String::ONE_BYTE_ENCODING); - Goto(&done_native); - - BIND(&two_byte); - { - // Two-byte sequential string case - result = AllocateSeqTwoByteString(new_length); - CopyStringCharacters(var_left.value(), result.value(), IntPtrConstant(0), - IntPtrConstant(0), word_left_length, - String::TWO_BYTE_ENCODING, - String::TWO_BYTE_ENCODING); - CopyStringCharacters(var_right.value(), result.value(), IntPtrConstant(0), - word_left_length, word_right_length, - String::TWO_BYTE_ENCODING, - String::TWO_BYTE_ENCODING); - Goto(&done_native); - } - - BIND(&slow); - { - // Try to unwrap indirect strings, restart the above attempt on success. - MaybeDerefIndirectStrings(&var_left, left_instance_type, &var_right, - right_instance_type, &non_cons); - Goto(&runtime); - } - } - BIND(&runtime); - { - result = CAST(CallRuntime(Runtime::kStringAdd, context, left, right)); - Goto(&done); - } - - BIND(&done_native); - { - IncrementCounter(counters->string_add_native(), 1); - Goto(&done); - } - - BIND(&done); - return result.value(); -} - -TNode<String> CodeStubAssembler::StringFromSingleUTF16EncodedCodePoint( - TNode<Int32T> codepoint) { - VARIABLE(var_result, MachineRepresentation::kTagged, EmptyStringConstant()); - - Label if_isword16(this), if_isword32(this), return_result(this); - - Branch(Uint32LessThan(codepoint, Int32Constant(0x10000)), &if_isword16, - &if_isword32); - - BIND(&if_isword16); - { - var_result.Bind(StringFromSingleCharCode(codepoint)); - Goto(&return_result); - } - - BIND(&if_isword32); - { - TNode<String> value = AllocateSeqTwoByteString(2); - StoreNoWriteBarrier( - MachineRepresentation::kWord32, value, - IntPtrConstant(SeqTwoByteString::kHeaderSize - kHeapObjectTag), - codepoint); - var_result.Bind(value); - Goto(&return_result); - } - - BIND(&return_result); - return CAST(var_result.value()); -} - TNode<Number> CodeStubAssembler::StringToNumber(TNode<String> input) { Label runtime(this, Label::kDeferred); Label end(this); @@ -7585,22 +7051,22 @@ TNode<Number> CodeStubAssembler::StringToNumber(TNode<String> input) { return var_result.value(); } -TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { +TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input, + Label* bailout) { TVARIABLE(String, result); TVARIABLE(Smi, smi_input); - Label runtime(this, Label::kDeferred), if_smi(this), if_heap_number(this), - done(this, &result); + Label if_smi(this), if_heap_number(this), done(this, &result); // Load the number string cache. TNode<FixedArray> number_string_cache = NumberStringCacheConstant(); // Make the hash mask from the length of the number string cache. It // contains two elements (number and string) for each cache entry. - // TODO(ishell): cleanup mask handling. - TNode<IntPtrT> mask = - BitcastTaggedSignedToWord(LoadFixedArrayBaseLength(number_string_cache)); - TNode<IntPtrT> one = IntPtrConstant(1); - mask = IntPtrSub(mask, one); + TNode<IntPtrT> number_string_cache_length = + LoadAndUntagFixedArrayBaseLength(number_string_cache); + TNode<Int32T> one = Int32Constant(1); + TNode<Word32T> mask = Int32Sub( + Word32Shr(TruncateWordToInt32(number_string_cache_length), one), one); GotoIfNot(TaggedIsSmi(input), &if_heap_number); smi_input = CAST(input); @@ -7611,36 +7077,35 @@ TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { Comment("NumberToString - HeapNumber"); TNode<HeapNumber> heap_number_input = CAST(input); // Try normalizing the HeapNumber. - TryHeapNumberToSmi(heap_number_input, smi_input, &if_smi); + TryHeapNumberToSmi(heap_number_input, &smi_input, &if_smi); // Make a hash from the two 32-bit values of the double. TNode<Int32T> low = LoadObjectField<Int32T>(heap_number_input, HeapNumber::kValueOffset); TNode<Int32T> high = LoadObjectField<Int32T>( heap_number_input, HeapNumber::kValueOffset + kIntSize); - TNode<Word32T> hash = Word32Xor(low, high); - TNode<IntPtrT> word_hash = WordShl(ChangeInt32ToIntPtr(hash), one); - TNode<WordT> index = - WordAnd(word_hash, WordSar(mask, SmiShiftBitsConstant())); + TNode<Word32T> hash = Word32And(Word32Xor(low, high), mask); + TNode<IntPtrT> entry_index = + Signed(ChangeUint32ToWord(Int32Add(hash, hash))); // Cache entry's key must be a heap number TNode<Object> number_key = - UnsafeLoadFixedArrayElement(number_string_cache, index); - GotoIf(TaggedIsSmi(number_key), &runtime); + UnsafeLoadFixedArrayElement(number_string_cache, entry_index); + GotoIf(TaggedIsSmi(number_key), bailout); TNode<HeapObject> number_key_heap_object = CAST(number_key); - GotoIfNot(IsHeapNumber(number_key_heap_object), &runtime); + GotoIfNot(IsHeapNumber(number_key_heap_object), bailout); // Cache entry's key must match the heap number value we're looking for. TNode<Int32T> low_compare = LoadObjectField<Int32T>( number_key_heap_object, HeapNumber::kValueOffset); TNode<Int32T> high_compare = LoadObjectField<Int32T>( number_key_heap_object, HeapNumber::kValueOffset + kIntSize); - GotoIfNot(Word32Equal(low, low_compare), &runtime); - GotoIfNot(Word32Equal(high, high_compare), &runtime); + GotoIfNot(Word32Equal(low, low_compare), bailout); + GotoIfNot(Word32Equal(high, high_compare), bailout); // Heap number match, return value from cache entry. - result = CAST( - UnsafeLoadFixedArrayElement(number_string_cache, index, kTaggedSize)); + result = CAST(UnsafeLoadFixedArrayElement(number_string_cache, entry_index, + kTaggedSize)); Goto(&done); } @@ -7648,17 +7113,28 @@ TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { { Comment("NumberToString - Smi"); // Load the smi key, make sure it matches the smi we're looking for. - TNode<Object> smi_index = BitcastWordToTagged(WordAnd( - WordShl(BitcastTaggedSignedToWord(smi_input.value()), one), mask)); + TNode<Word32T> hash = Word32And(SmiToInt32(smi_input.value()), mask); + TNode<IntPtrT> entry_index = + Signed(ChangeUint32ToWord(Int32Add(hash, hash))); TNode<Object> smi_key = UnsafeLoadFixedArrayElement( - number_string_cache, smi_index, 0, SMI_PARAMETERS); - GotoIf(TaggedNotEqual(smi_key, smi_input.value()), &runtime); + number_string_cache, entry_index, 0, INTPTR_PARAMETERS); + GotoIf(TaggedNotEqual(smi_key, smi_input.value()), bailout); // Smi match, return value from cache entry. - result = CAST(UnsafeLoadFixedArrayElement(number_string_cache, smi_index, - kTaggedSize, SMI_PARAMETERS)); + result = CAST(UnsafeLoadFixedArrayElement(number_string_cache, entry_index, + kTaggedSize, INTPTR_PARAMETERS)); Goto(&done); } + BIND(&done); + return result.value(); +} + +TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { + TVARIABLE(String, result); + Label runtime(this, Label::kDeferred), done(this, &result); + + result = NumberToString(input, &runtime); + Goto(&done); BIND(&runtime); { @@ -8290,102 +7766,129 @@ void CodeStubAssembler::DecrementCounter(StatsCounter* counter, int delta) { } } -void CodeStubAssembler::Increment(Variable* variable, int value, - ParameterMode mode) { - DCHECK_IMPLIES(mode == INTPTR_PARAMETERS, - variable->rep() == MachineType::PointerRepresentation()); - DCHECK_IMPLIES(mode == SMI_PARAMETERS, CanBeTaggedSigned(variable->rep())); - variable->Bind(IntPtrOrSmiAdd(variable->value(), - IntPtrOrSmiConstant(value, mode), mode)); +template <typename TIndex> +void CodeStubAssembler::Increment(TVariable<TIndex>* variable, int value) { + *variable = + IntPtrOrSmiAdd(variable->value(), IntPtrOrSmiConstant<TIndex>(value)); } +// Instantiate Increment for Smi and IntPtrT. +// TODO(v8:9708): Consider renaming to [Smi|IntPtrT|RawPtrT]Increment. +template void CodeStubAssembler::Increment<Smi>(TVariable<Smi>* variable, + int value); +template void CodeStubAssembler::Increment<IntPtrT>( + TVariable<IntPtrT>* variable, int value); +template void CodeStubAssembler::Increment<RawPtrT>( + TVariable<RawPtrT>* variable, int value); + void CodeStubAssembler::Use(Label* label) { GotoIf(Word32Equal(Int32Constant(0), Int32Constant(1)), label); } -void CodeStubAssembler::TryToName(Node* key, Label* if_keyisindex, - Variable* var_index, Label* if_keyisunique, - Variable* var_unique, Label* if_bailout, +void CodeStubAssembler::TryToName(SloppyTNode<Object> key, Label* if_keyisindex, + TVariable<IntPtrT>* var_index, + Label* if_keyisunique, + TVariable<Name>* var_unique, + Label* if_bailout, Label* if_notinternalized) { - DCHECK_EQ(MachineType::PointerRepresentation(), var_index->rep()); - DCHECK_EQ(MachineRepresentation::kTagged, var_unique->rep()); Comment("TryToName"); - Label if_hascachedindex(this), if_keyisnotindex(this), if_thinstring(this), - if_keyisother(this, Label::kDeferred); + Label if_keyisnotindex(this); // Handle Smi and HeapNumber keys. - var_index->Bind(TryToIntptr(key, &if_keyisnotindex)); + *var_index = TryToIntptr(key, &if_keyisnotindex); Goto(if_keyisindex); BIND(&if_keyisnotindex); - TNode<Map> key_map = LoadMap(key); - var_unique->Bind(key); - // Symbols are unique. - GotoIf(IsSymbolMap(key_map), if_keyisunique); - TNode<Uint16T> key_instance_type = LoadMapInstanceType(key_map); - // Miss if |key| is not a String. - STATIC_ASSERT(FIRST_NAME_TYPE == FIRST_TYPE); - GotoIfNot(IsStringInstanceType(key_instance_type), &if_keyisother); - - // |key| is a String. Check if it has a cached array index. - TNode<Uint32T> hash = LoadNameHashField(key); - GotoIf(IsClearWord32(hash, Name::kDoesNotContainCachedArrayIndexMask), - &if_hascachedindex); - // No cached array index. If the string knows that it contains an index, - // then it must be an uncacheable index. Handle this case in the runtime. - GotoIf(IsClearWord32(hash, Name::kIsNotArrayIndexMask), if_bailout); - // Check if we have a ThinString. - GotoIf(InstanceTypeEqual(key_instance_type, THIN_STRING_TYPE), - &if_thinstring); - GotoIf(InstanceTypeEqual(key_instance_type, THIN_ONE_BYTE_STRING_TYPE), - &if_thinstring); - // Finally, check if |key| is internalized. - STATIC_ASSERT(kNotInternalizedTag != 0); - GotoIf(IsSetWord32(key_instance_type, kIsNotInternalizedMask), - if_notinternalized != nullptr ? if_notinternalized : if_bailout); - Goto(if_keyisunique); + { + Label if_symbol(this), if_string(this), + if_keyisother(this, Label::kDeferred); + TNode<HeapObject> key_heap_object = CAST(key); + TNode<Map> key_map = LoadMap(key_heap_object); - BIND(&if_thinstring); - var_unique->Bind( - LoadObjectField<String>(CAST(key), ThinString::kActualOffset)); - Goto(if_keyisunique); + GotoIf(IsSymbolMap(key_map), &if_symbol); - BIND(&if_hascachedindex); - var_index->Bind(DecodeWordFromWord32<Name::ArrayIndexValueBits>(hash)); - Goto(if_keyisindex); + // Miss if |key| is not a String. + STATIC_ASSERT(FIRST_NAME_TYPE == FIRST_TYPE); + TNode<Uint16T> key_instance_type = LoadMapInstanceType(key_map); + Branch(IsStringInstanceType(key_instance_type), &if_string, &if_keyisother); + + // Symbols are unique. + BIND(&if_symbol); + { + *var_unique = CAST(key); + Goto(if_keyisunique); + } - BIND(&if_keyisother); - GotoIfNot(InstanceTypeEqual(key_instance_type, ODDBALL_TYPE), if_bailout); - var_unique->Bind(LoadObjectField(key, Oddball::kToStringOffset)); - Goto(if_keyisunique); + BIND(&if_string); + { + Label if_hascachedindex(this), if_thinstring(this); + + // |key| is a String. Check if it has a cached array index. + TNode<String> key_string = CAST(key); + TNode<Uint32T> hash = LoadNameHashField(key_string); + GotoIf(IsClearWord32(hash, Name::kDoesNotContainCachedArrayIndexMask), + &if_hascachedindex); + // No cached array index. If the string knows that it contains an index, + // then it must be an uncacheable index. Handle this case in the runtime. + GotoIf(IsClearWord32(hash, Name::kIsNotArrayIndexMask), if_bailout); + // Check if we have a ThinString. + GotoIf(InstanceTypeEqual(key_instance_type, THIN_STRING_TYPE), + &if_thinstring); + GotoIf(InstanceTypeEqual(key_instance_type, THIN_ONE_BYTE_STRING_TYPE), + &if_thinstring); + // Finally, check if |key| is internalized. + STATIC_ASSERT(kNotInternalizedTag != 0); + GotoIf(IsSetWord32(key_instance_type, kIsNotInternalizedMask), + if_notinternalized != nullptr ? if_notinternalized : if_bailout); + + *var_unique = key_string; + Goto(if_keyisunique); + + BIND(&if_thinstring); + *var_unique = + LoadObjectField<String>(key_string, ThinString::kActualOffset); + Goto(if_keyisunique); + + BIND(&if_hascachedindex); + *var_index = + Signed(DecodeWordFromWord32<Name::ArrayIndexValueBits>(hash)); + Goto(if_keyisindex); + } + + BIND(&if_keyisother); + { + GotoIfNot(InstanceTypeEqual(key_instance_type, ODDBALL_TYPE), if_bailout); + *var_unique = + LoadObjectField<String>(key_heap_object, Oddball::kToStringOffset); + Goto(if_keyisunique); + } + } } void CodeStubAssembler::TryInternalizeString( - Node* string, Label* if_index, Variable* var_index, Label* if_internalized, - Variable* var_internalized, Label* if_not_internalized, Label* if_bailout) { - DCHECK(var_index->rep() == MachineType::PointerRepresentation()); - DCHECK_EQ(var_internalized->rep(), MachineRepresentation::kTagged); - CSA_SLOW_ASSERT(this, IsString(string)); + SloppyTNode<String> string, Label* if_index, TVariable<IntPtrT>* var_index, + Label* if_internalized, TVariable<Name>* var_internalized, + Label* if_not_internalized, Label* if_bailout) { TNode<ExternalReference> function = ExternalConstant(ExternalReference::try_internalize_string_function()); TNode<ExternalReference> const isolate_ptr = ExternalConstant(ExternalReference::isolate_address(isolate())); - Node* result = - CallCFunction(function, MachineType::AnyTagged(), - std::make_pair(MachineType::Pointer(), isolate_ptr), - std::make_pair(MachineType::AnyTagged(), string)); + TNode<Object> result = + CAST(CallCFunction(function, MachineType::AnyTagged(), + std::make_pair(MachineType::Pointer(), isolate_ptr), + std::make_pair(MachineType::AnyTagged(), string))); Label internalized(this); GotoIf(TaggedIsNotSmi(result), &internalized); - TNode<IntPtrT> word_result = SmiUntag(result); + TNode<IntPtrT> word_result = SmiUntag(CAST(result)); GotoIf(IntPtrEqual(word_result, IntPtrConstant(ResultSentinel::kNotFound)), if_not_internalized); GotoIf(IntPtrEqual(word_result, IntPtrConstant(ResultSentinel::kUnsupported)), if_bailout); - var_index->Bind(word_result); + *var_index = word_result; Goto(if_index); BIND(&internalized); - var_internalized->Bind(result); + *var_internalized = CAST(result); Goto(if_internalized); } @@ -8712,31 +8215,6 @@ TNode<Object> CodeStubAssembler::BasicLoadNumberDictionaryElement( return LoadValueByKeyIndex<NumberDictionary>(dictionary, index); } -void CodeStubAssembler::BasicStoreNumberDictionaryElement( - TNode<NumberDictionary> dictionary, TNode<IntPtrT> intptr_index, - TNode<Object> value, Label* not_data, Label* if_hole, Label* read_only) { - TVARIABLE(IntPtrT, var_entry); - Label if_found(this); - NumberDictionaryLookup(dictionary, intptr_index, &if_found, &var_entry, - if_hole); - BIND(&if_found); - - // Check that the value is a data property. - TNode<IntPtrT> index = EntryToIndex<NumberDictionary>(var_entry.value()); - TNode<Uint32T> details = - LoadDetailsByKeyIndex<NumberDictionary>(dictionary, index); - TNode<Uint32T> kind = DecodeWord32<PropertyDetails::KindField>(details); - // TODO(jkummerow): Support accessors without missing? - GotoIfNot(Word32Equal(kind, Int32Constant(kData)), not_data); - - // Check that the property is writeable. - GotoIf(IsSetWord32(details, PropertyDetails::kAttributesReadOnlyMask), - read_only); - - // Finally, store the value. - StoreValueByKeyIndex<NumberDictionary>(dictionary, index, value); -} - template <class Dictionary> void CodeStubAssembler::FindInsertionEntry(TNode<Dictionary> dictionary, TNode<Name> key, @@ -8858,16 +8336,16 @@ void CodeStubAssembler::LookupLinear(TNode<Name> unique_name, first_inclusive, IntPtrMul(ChangeInt32ToIntPtr(number_of_valid_entries), factor)); - BuildFastLoop( + BuildFastLoop<IntPtrT>( last_exclusive, first_inclusive, - [=](SloppyTNode<IntPtrT> name_index) { + [=](TNode<IntPtrT> name_index) { TNode<MaybeObject> element = LoadArrayElement(array, Array::kHeaderSize, name_index); TNode<Name> candidate_name = CAST(element); *var_name_index = name_index; GotoIf(TaggedEqual(candidate_name, unique_name), if_found); }, - -Array::kEntrySize, INTPTR_PARAMETERS, IndexAdvanceMode::kPre); + -Array::kEntrySize, IndexAdvanceMode::kPre); Goto(if_not_found); } @@ -9029,7 +8507,7 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( TNode<Uint16T> type = LoadMapInstanceType(map); TNode<Uint32T> bit_field3 = EnsureOnlyHasSimpleProperties(map, type, bailout); - TNode<DescriptorArray> descriptors = LoadMapDescriptors(map); + TVARIABLE(DescriptorArray, var_descriptors, LoadMapDescriptors(map)); TNode<Uint32T> nof_descriptors = DecodeWord32<Map::NumberOfOwnDescriptorsBits>(bit_field3); @@ -9044,25 +8522,23 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( // Note: var_end_key_index is exclusive for the loop TVARIABLE(IntPtrT, var_end_key_index, ToKeyIndex<DescriptorArray>(nof_descriptors)); - VariableList list( - {&var_stable, &var_has_symbol, &var_is_symbol_processing_loop, - &var_start_key_index, &var_end_key_index}, - zone()); + VariableList list({&var_descriptors, &var_stable, &var_has_symbol, + &var_is_symbol_processing_loop, &var_start_key_index, + &var_end_key_index}, + zone()); Label descriptor_array_loop( - this, {&var_stable, &var_has_symbol, &var_is_symbol_processing_loop, - &var_start_key_index, &var_end_key_index}); + this, {&var_descriptors, &var_stable, &var_has_symbol, + &var_is_symbol_processing_loop, &var_start_key_index, + &var_end_key_index}); Goto(&descriptor_array_loop); BIND(&descriptor_array_loop); - BuildFastLoop( + BuildFastLoop<IntPtrT>( list, var_start_key_index.value(), var_end_key_index.value(), - [=, &var_stable, &var_has_symbol, &var_is_symbol_processing_loop, - &var_start_key_index, &var_end_key_index](Node* index) { - TNode<IntPtrT> descriptor_key_index = - TNode<IntPtrT>::UncheckedCast(index); + [&](TNode<IntPtrT> descriptor_key_index) { TNode<Name> next_key = - LoadKeyByKeyIndex(descriptors, descriptor_key_index); + LoadKeyByKeyIndex(var_descriptors.value(), descriptor_key_index); TVARIABLE(Object, var_value, SmiConstant(0)); Label callback(this), next_iteration(this); @@ -9117,7 +8593,7 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( // Directly decode from the descriptor array if |object| did not // change shape. var_map = map; - var_meta_storage = descriptors; + var_meta_storage = var_descriptors.value(); var_entry = Signed(descriptor_key_index); Goto(&if_found_fast); } @@ -9183,19 +8659,21 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( BIND(&callback); body(next_key, var_value.value()); - // Check if |object| is still stable, i.e. we can proceed using - // property details from preloaded |descriptors|. - var_stable = Select<BoolT>( - var_stable.value(), - [=] { return TaggedEqual(LoadMap(object), map); }, - [=] { return Int32FalseConstant(); }); + // Check if |object| is still stable, i.e. the descriptors in the + // preloaded |descriptors| are still the same modulo in-place + // representation changes. + GotoIfNot(var_stable.value(), &next_iteration); + var_stable = TaggedEqual(LoadMap(object), map); + // Reload the descriptors just in case the actual array changed, and + // any of the field representations changed in-place. + var_descriptors = LoadMapDescriptors(map); Goto(&next_iteration); } } BIND(&next_iteration); }, - DescriptorArray::kEntrySize, INTPTR_PARAMETERS, IndexAdvanceMode::kPost); + DescriptorArray::kEntrySize, IndexAdvanceMode::kPost); if (mode == kEnumerationOrder) { Label done(this); @@ -9205,14 +8683,73 @@ void CodeStubAssembler::ForEachEnumerableOwnProperty( var_is_symbol_processing_loop = Int32TrueConstant(); // Add DescriptorArray::kEntrySize to make the var_end_key_index exclusive // as BuildFastLoop() expects. - Increment(&var_end_key_index, DescriptorArray::kEntrySize, - INTPTR_PARAMETERS); + Increment(&var_end_key_index, DescriptorArray::kEntrySize); Goto(&descriptor_array_loop); BIND(&done); } } +TNode<Object> CodeStubAssembler::GetConstructor(TNode<Map> map) { + TVARIABLE(HeapObject, var_maybe_constructor); + var_maybe_constructor = map; + Label loop(this, &var_maybe_constructor), done(this); + GotoIfNot(IsMap(var_maybe_constructor.value()), &done); + Goto(&loop); + + BIND(&loop); + { + var_maybe_constructor = CAST(LoadObjectField( + var_maybe_constructor.value(), Map::kConstructorOrBackPointerOffset)); + GotoIf(IsMap(var_maybe_constructor.value()), &loop); + Goto(&done); + } + + BIND(&done); + return var_maybe_constructor.value(); +} + +TNode<NativeContext> CodeStubAssembler::GetCreationContext( + TNode<JSReceiver> receiver, Label* if_bailout) { + TNode<Map> receiver_map = LoadMap(receiver); + TNode<Object> constructor = GetConstructor(receiver_map); + + TVARIABLE(JSFunction, var_function); + + Label done(this), if_jsfunction(this), if_jsgenerator(this); + GotoIf(TaggedIsSmi(constructor), if_bailout); + + TNode<Map> function_map = LoadMap(CAST(constructor)); + GotoIf(IsJSFunctionMap(function_map), &if_jsfunction); + GotoIf(IsJSGeneratorMap(function_map), &if_jsgenerator); + // Remote objects don't have a creation context. + GotoIf(IsFunctionTemplateInfoMap(function_map), if_bailout); + + CSA_ASSERT(this, IsJSFunctionMap(receiver_map)); + var_function = CAST(receiver); + Goto(&done); + + BIND(&if_jsfunction); + { + var_function = CAST(constructor); + Goto(&done); + } + + BIND(&if_jsgenerator); + { + var_function = LoadJSGeneratorObjectFunction(CAST(receiver)); + Goto(&done); + } + + BIND(&done); + TNode<Context> context = LoadJSFunctionContext(var_function.value()); + + GotoIfNot(IsContext(context), if_bailout); + + TNode<NativeContext> native_context = LoadNativeContext(context); + return native_context; +} + void CodeStubAssembler::DescriptorLookup( SloppyTNode<Name> unique_name, SloppyTNode<DescriptorArray> descriptors, SloppyTNode<Uint32T> bitfield3, Label* if_found, @@ -9302,7 +8839,7 @@ void CodeStubAssembler::TryLookupPropertyInSimpleObject( } void CodeStubAssembler::TryLookupProperty( - SloppyTNode<JSObject> object, SloppyTNode<Map> map, + SloppyTNode<JSReceiver> object, SloppyTNode<Map> map, SloppyTNode<Int32T> instance_type, SloppyTNode<Name> unique_name, Label* if_found_fast, Label* if_found_dict, Label* if_found_global, TVariable<HeapObject>* var_meta_storage, TVariable<IntPtrT>* var_name_index, @@ -9310,7 +8847,7 @@ void CodeStubAssembler::TryLookupProperty( Label if_objectisspecial(this); GotoIf(IsSpecialReceiverInstanceType(instance_type), &if_objectisspecial); - TryLookupPropertyInSimpleObject(object, map, unique_name, if_found_fast, + TryLookupPropertyInSimpleObject(CAST(object), map, unique_name, if_found_fast, if_found_dict, var_meta_storage, var_name_index, if_not_found); @@ -9547,25 +9084,44 @@ TNode<Object> CodeStubAssembler::CallGetterIfAccessor( // AccessorPair case. { if (mode == kCallJSGetter) { + Label if_callable(this), if_function_template_info(this); Node* accessor_pair = value; TNode<HeapObject> getter = CAST(LoadObjectField(accessor_pair, AccessorPair::kGetterOffset)); TNode<Map> getter_map = LoadMap(getter); - TNode<Uint16T> instance_type = LoadMapInstanceType(getter_map); - // FunctionTemplateInfo getters are not supported yet. - GotoIf(InstanceTypeEqual(instance_type, FUNCTION_TEMPLATE_INFO_TYPE), - if_bailout); + + GotoIf(IsCallableMap(getter_map), &if_callable); + GotoIf(IsFunctionTemplateInfoMap(getter_map), &if_function_template_info); // Return undefined if the {getter} is not callable. var_value.Bind(UndefinedConstant()); - GotoIfNot(IsCallableMap(getter_map), &done); + Goto(&done); + + BIND(&if_callable); + { + // Call the accessor. + Callable callable = CodeFactory::Call(isolate()); + Node* result = CallJS(callable, context, getter, receiver); + var_value.Bind(result); + Goto(&done); + } - // Call the accessor. - Callable callable = CodeFactory::Call(isolate()); - Node* result = CallJS(callable, context, getter, receiver); - var_value.Bind(result); + BIND(&if_function_template_info); + { + TNode<HeapObject> cached_property_name = LoadObjectField<HeapObject>( + getter, FunctionTemplateInfo::kCachedPropertyNameOffset); + GotoIfNot(IsTheHole(cached_property_name), if_bailout); + + TNode<NativeContext> creation_context = + GetCreationContext(CAST(receiver), if_bailout); + var_value.Bind(CallBuiltin( + Builtins::kCallFunctionTemplate_CheckAccessAndCompatibleReceiver, + creation_context, getter, IntPtrConstant(0), receiver)); + Goto(&done); + } + } else { + Goto(&done); } - Goto(&done); } // AccessorInfo case. @@ -9617,10 +9173,11 @@ TNode<Object> CodeStubAssembler::CallGetterIfAccessor( GotoIfNot(IsLengthString( LoadObjectField(accessor_info, AccessorInfo::kNameOffset)), if_bailout); - Node* receiver_value = LoadJSPrimitiveWrapperValue(receiver); + TNode<Object> receiver_value = + LoadJSPrimitiveWrapperValue(CAST(receiver)); GotoIfNot(TaggedIsNotSmi(receiver_value), if_bailout); - GotoIfNot(IsString(receiver_value), if_bailout); - var_value.Bind(LoadStringLengthAsSmi(receiver_value)); + GotoIfNot(IsString(CAST(receiver_value)), if_bailout); + var_value.Bind(LoadStringLengthAsSmi(CAST(receiver_value))); Goto(&done); } } @@ -9808,18 +9365,14 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, } BIND(&if_isfaststringwrapper); { - CSA_ASSERT(this, HasInstanceType(object, JS_PRIMITIVE_WRAPPER_TYPE)); - Node* string = LoadJSPrimitiveWrapperValue(object); - CSA_ASSERT(this, IsString(string)); + TNode<String> string = CAST(LoadJSPrimitiveWrapperValue(CAST(object))); TNode<IntPtrT> length = LoadStringLengthAsWord(string); GotoIf(UintPtrLessThan(intptr_index, length), if_found); Goto(&if_isobjectorsmi); } BIND(&if_isslowstringwrapper); { - CSA_ASSERT(this, HasInstanceType(object, JS_PRIMITIVE_WRAPPER_TYPE)); - Node* string = LoadJSPrimitiveWrapperValue(object); - CSA_ASSERT(this, IsString(string)); + TNode<String> string = CAST(LoadJSPrimitiveWrapperValue(CAST(object))); TNode<IntPtrT> length = LoadStringLengthAsWord(string); GotoIf(UintPtrLessThan(intptr_index, length), if_found); Goto(&if_isdictionary); @@ -9892,8 +9445,8 @@ void CodeStubAssembler::TryPrototypeChainLookup( GotoIf(InstanceTypeEqual(instance_type, JS_PROXY_TYPE), if_proxy); } - VARIABLE(var_index, MachineType::PointerRepresentation()); - VARIABLE(var_unique, MachineRepresentation::kTagged); + TVARIABLE(IntPtrT, var_index); + TVARIABLE(Name, var_unique); Label if_keyisindex(this), if_iskeyunique(this); TryToName(key, &if_keyisindex, &var_index, &if_iskeyunique, &var_unique, @@ -9905,9 +9458,7 @@ void CodeStubAssembler::TryPrototypeChainLookup( TVARIABLE(Map, var_holder_map, map); TVARIABLE(Int32T, var_holder_instance_type, instance_type); - VariableList merged_variables( - {&var_holder, &var_holder_map, &var_holder_instance_type}, zone()); - Label loop(this, merged_variables); + Label loop(this, {&var_holder, &var_holder_map, &var_holder_instance_type}); Goto(&loop); BIND(&loop); { @@ -9950,9 +9501,7 @@ void CodeStubAssembler::TryPrototypeChainLookup( TVARIABLE(Map, var_holder_map, map); TVARIABLE(Int32T, var_holder_instance_type, instance_type); - VariableList merged_variables( - {&var_holder, &var_holder_map, &var_holder_instance_type}, zone()); - Label loop(this, merged_variables); + Label loop(this, {&var_holder, &var_holder_map, &var_holder_instance_type}); Goto(&loop); BIND(&loop); { @@ -9978,22 +9527,22 @@ void CodeStubAssembler::TryPrototypeChainLookup( } } -Node* CodeStubAssembler::HasInPrototypeChain(Node* context, Node* object, - SloppyTNode<Object> prototype) { - CSA_ASSERT(this, TaggedIsNotSmi(object)); - VARIABLE(var_result, MachineRepresentation::kTagged); +TNode<Oddball> CodeStubAssembler::HasInPrototypeChain(TNode<Context> context, + TNode<HeapObject> object, + TNode<Object> prototype) { + TVARIABLE(Oddball, var_result); Label return_false(this), return_true(this), return_runtime(this, Label::kDeferred), return_result(this); // Loop through the prototype chain looking for the {prototype}. - VARIABLE(var_object_map, MachineRepresentation::kTagged, LoadMap(object)); + TVARIABLE(Map, var_object_map, LoadMap(object)); Label loop(this, &var_object_map); Goto(&loop); BIND(&loop); { // Check if we can determine the prototype directly from the {object_map}. Label if_objectisdirect(this), if_objectisspecial(this, Label::kDeferred); - Node* object_map = var_object_map.value(); + TNode<Map> object_map = var_object_map.value(); TNode<Uint16T> object_instance_type = LoadMapInstanceType(object_map); Branch(IsSpecialReceiverInstanceType(object_instance_type), &if_objectisspecial, &if_objectisdirect); @@ -10018,22 +9567,22 @@ Node* CodeStubAssembler::HasInPrototypeChain(Node* context, Node* object, // Continue with the prototype. CSA_ASSERT(this, TaggedIsNotSmi(object_prototype)); - var_object_map.Bind(LoadMap(object_prototype)); + var_object_map = LoadMap(object_prototype); Goto(&loop); } BIND(&return_true); - var_result.Bind(TrueConstant()); + var_result = TrueConstant(); Goto(&return_result); BIND(&return_false); - var_result.Bind(FalseConstant()); + var_result = FalseConstant(); Goto(&return_result); BIND(&return_runtime); { // Fallback to the runtime implementation. - var_result.Bind( + var_result = CAST( CallRuntime(Runtime::kHasInPrototypeChain, context, object, prototype)); } Goto(&return_result); @@ -10042,63 +9591,67 @@ Node* CodeStubAssembler::HasInPrototypeChain(Node* context, Node* object, return var_result.value(); } -Node* CodeStubAssembler::OrdinaryHasInstance(Node* context, Node* callable, - Node* object) { - VARIABLE(var_result, MachineRepresentation::kTagged); +TNode<Oddball> CodeStubAssembler::OrdinaryHasInstance( + TNode<Context> context, TNode<Object> callable_maybe_smi, + TNode<Object> object_maybe_smi) { + TVARIABLE(Oddball, var_result); Label return_runtime(this, Label::kDeferred), return_result(this); GotoIfForceSlowPath(&return_runtime); // Goto runtime if {object} is a Smi. - GotoIf(TaggedIsSmi(object), &return_runtime); + GotoIf(TaggedIsSmi(object_maybe_smi), &return_runtime); // Goto runtime if {callable} is a Smi. - GotoIf(TaggedIsSmi(callable), &return_runtime); - - // Load map of {callable}. - TNode<Map> callable_map = LoadMap(callable); - - // Goto runtime if {callable} is not a JSFunction. - TNode<Uint16T> callable_instance_type = LoadMapInstanceType(callable_map); - GotoIfNot(InstanceTypeEqual(callable_instance_type, JS_FUNCTION_TYPE), - &return_runtime); + GotoIf(TaggedIsSmi(callable_maybe_smi), &return_runtime); - GotoIfPrototypeRequiresRuntimeLookup(CAST(callable), callable_map, - &return_runtime); - - // Get the "prototype" (or initial map) of the {callable}. - TNode<HeapObject> callable_prototype = LoadObjectField<HeapObject>( - CAST(callable), JSFunction::kPrototypeOrInitialMapOffset); { - Label no_initial_map(this), walk_prototype_chain(this); - TVARIABLE(HeapObject, var_callable_prototype, callable_prototype); + // Load map of {callable}. + TNode<HeapObject> object = CAST(object_maybe_smi); + TNode<HeapObject> callable = CAST(callable_maybe_smi); + TNode<Map> callable_map = LoadMap(callable); - // Resolve the "prototype" if the {callable} has an initial map. - GotoIfNot(IsMap(callable_prototype), &no_initial_map); - var_callable_prototype = - LoadObjectField<HeapObject>(callable_prototype, Map::kPrototypeOffset); - Goto(&walk_prototype_chain); + // Goto runtime if {callable} is not a JSFunction. + TNode<Uint16T> callable_instance_type = LoadMapInstanceType(callable_map); + GotoIfNot(InstanceTypeEqual(callable_instance_type, JS_FUNCTION_TYPE), + &return_runtime); - BIND(&no_initial_map); - // {callable_prototype} is the hole if the "prototype" property hasn't been - // requested so far. - Branch(TaggedEqual(callable_prototype, TheHoleConstant()), &return_runtime, - &walk_prototype_chain); + GotoIfPrototypeRequiresRuntimeLookup(CAST(callable), callable_map, + &return_runtime); - BIND(&walk_prototype_chain); - callable_prototype = var_callable_prototype.value(); - } + // Get the "prototype" (or initial map) of the {callable}. + TNode<HeapObject> callable_prototype = LoadObjectField<HeapObject>( + callable, JSFunction::kPrototypeOrInitialMapOffset); + { + Label no_initial_map(this), walk_prototype_chain(this); + TVARIABLE(HeapObject, var_callable_prototype, callable_prototype); + + // Resolve the "prototype" if the {callable} has an initial map. + GotoIfNot(IsMap(callable_prototype), &no_initial_map); + var_callable_prototype = LoadObjectField<HeapObject>( + callable_prototype, Map::kPrototypeOffset); + Goto(&walk_prototype_chain); + + BIND(&no_initial_map); + // {callable_prototype} is the hole if the "prototype" property hasn't + // been requested so far. + Branch(TaggedEqual(callable_prototype, TheHoleConstant()), + &return_runtime, &walk_prototype_chain); + + BIND(&walk_prototype_chain); + callable_prototype = var_callable_prototype.value(); + } - // Loop through the prototype chain looking for the {callable} prototype. - CSA_ASSERT(this, IsJSReceiver(callable_prototype)); - var_result.Bind(HasInPrototypeChain(context, object, callable_prototype)); - Goto(&return_result); + // Loop through the prototype chain looking for the {callable} prototype. + var_result = HasInPrototypeChain(context, object, callable_prototype); + Goto(&return_result); + } BIND(&return_runtime); { // Fallback to the runtime implementation. - var_result.Bind( - CallRuntime(Runtime::kOrdinaryHasInstance, context, callable, object)); + var_result = CAST(CallRuntime(Runtime::kOrdinaryHasInstance, context, + callable_maybe_smi, object_maybe_smi)); } Goto(&return_result); @@ -10111,34 +9664,72 @@ TNode<IntPtrT> CodeStubAssembler::ElementOffsetFromIndex(Node* index_node, ParameterMode mode, int base_size) { CSA_SLOW_ASSERT(this, MatchesParameterMode(index_node, mode)); + if (mode == SMI_PARAMETERS) { + return ElementOffsetFromIndex(ReinterpretCast<Smi>(index_node), kind, + base_size); + } else { + DCHECK(mode == INTPTR_PARAMETERS); + return ElementOffsetFromIndex(ReinterpretCast<IntPtrT>(index_node), kind, + base_size); + } +} + +template <typename TIndex> +TNode<IntPtrT> CodeStubAssembler::ElementOffsetFromIndex( + TNode<TIndex> index_node, ElementsKind kind, int base_size) { + // TODO(v8:9708): Remove IntPtrT variant in favor of UintPtrT. + static_assert(std::is_same<TIndex, Smi>::value || + std::is_same<TIndex, IntPtrT>::value || + std::is_same<TIndex, UintPtrT>::value, + "Only Smi, UintPtrT or IntPtrT index nodes are allowed"); int element_size_shift = ElementsKindToShiftSize(kind); int element_size = 1 << element_size_shift; int const kSmiShiftBits = kSmiShiftSize + kSmiTagSize; intptr_t index = 0; + TNode<IntPtrT> intptr_index_node; bool constant_index = false; - if (mode == SMI_PARAMETERS) { + if (std::is_same<TIndex, Smi>::value) { + TNode<Smi> smi_index_node = ReinterpretCast<Smi>(index_node); element_size_shift -= kSmiShiftBits; Smi smi_index; - constant_index = ToSmiConstant(index_node, &smi_index); - if (constant_index) index = smi_index.value(); - index_node = BitcastTaggedSignedToWord(index_node); + constant_index = ToSmiConstant(smi_index_node, &smi_index); + if (constant_index) { + index = smi_index.value(); + } else { + if (COMPRESS_POINTERS_BOOL) { + smi_index_node = NormalizeSmiIndex(smi_index_node); + } + } + intptr_index_node = BitcastTaggedToWordForTagAndSmiBits(smi_index_node); } else { - DCHECK(mode == INTPTR_PARAMETERS); - constant_index = ToIntPtrConstant(index_node, &index); + intptr_index_node = ReinterpretCast<IntPtrT>(index_node); + constant_index = ToIntPtrConstant(intptr_index_node, &index); } if (constant_index) { return IntPtrConstant(base_size + element_size * index); } - TNode<WordT> shifted_index = + TNode<IntPtrT> shifted_index = (element_size_shift == 0) - ? UncheckedCast<WordT>(index_node) + ? intptr_index_node : ((element_size_shift > 0) - ? WordShl(index_node, IntPtrConstant(element_size_shift)) - : WordSar(index_node, IntPtrConstant(-element_size_shift))); + ? WordShl(intptr_index_node, + IntPtrConstant(element_size_shift)) + : WordSar(intptr_index_node, + IntPtrConstant(-element_size_shift))); return IntPtrAdd(IntPtrConstant(base_size), Signed(shifted_index)); } +// Instantiate ElementOffsetFromIndex for Smi and IntPtrT. +template V8_EXPORT_PRIVATE TNode<IntPtrT> +CodeStubAssembler::ElementOffsetFromIndex<Smi>(TNode<Smi> index_node, + ElementsKind kind, + int base_size); +template V8_EXPORT_PRIVATE TNode<IntPtrT> +CodeStubAssembler::ElementOffsetFromIndex<IntPtrT>(TNode<IntPtrT> index_node, + ElementsKind kind, + int base_size); + TNode<BoolT> CodeStubAssembler::IsOffsetInBounds(SloppyTNode<IntPtrT> offset, SloppyTNode<IntPtrT> length, int header_size, @@ -10146,8 +9737,7 @@ TNode<BoolT> CodeStubAssembler::IsOffsetInBounds(SloppyTNode<IntPtrT> offset, // Make sure we point to the last field. int element_size = 1 << ElementsKindToShiftSize(kind); int correction = header_size - kHeapObjectTag - element_size; - TNode<IntPtrT> last_offset = - ElementOffsetFromIndex(length, kind, INTPTR_PARAMETERS, correction); + TNode<IntPtrT> last_offset = ElementOffsetFromIndex(length, kind, correction); return IntPtrLessThanOrEqual(offset, last_offset); } @@ -10203,8 +9793,9 @@ TNode<FeedbackVector> CodeStubAssembler::LoadFeedbackVectorForStub() { return CAST(LoadFeedbackVector(function)); } -void CodeStubAssembler::UpdateFeedback(Node* feedback, Node* maybe_vector, - Node* slot_id) { +void CodeStubAssembler::UpdateFeedback(TNode<Smi> feedback, + TNode<HeapObject> maybe_vector, + TNode<UintPtrT> slot_id) { Label end(this); // If feedback_vector is not valid, then nothing to do. GotoIf(IsUndefined(maybe_vector), &end); @@ -10216,7 +9807,7 @@ void CodeStubAssembler::UpdateFeedback(Node* feedback, Node* maybe_vector, TNode<MaybeObject> feedback_element = LoadFeedbackVectorSlot(feedback_vector, slot_id); TNode<Smi> previous_feedback = CAST(feedback_element); - TNode<Smi> combined_feedback = SmiOr(previous_feedback, CAST(feedback)); + TNode<Smi> combined_feedback = SmiOr(previous_feedback, feedback); GotoIf(SmiEqual(previous_feedback, combined_feedback), &end); { @@ -10230,7 +9821,7 @@ void CodeStubAssembler::UpdateFeedback(Node* feedback, Node* maybe_vector, } void CodeStubAssembler::ReportFeedbackUpdate( - SloppyTNode<FeedbackVector> feedback_vector, SloppyTNode<IntPtrT> slot_id, + TNode<FeedbackVector> feedback_vector, SloppyTNode<UintPtrT> slot_id, const char* reason) { // Reset profiler ticks. StoreObjectFieldNoWriteBarrier( @@ -10241,7 +9832,7 @@ void CodeStubAssembler::ReportFeedbackUpdate( // Trace the update. CallRuntime(Runtime::kInterpreterTraceUpdateFeedback, NoContextConstant(), LoadFromParentFrame(JavaScriptFrameConstants::kFunctionOffset), - SmiTag(slot_id), StringConstant(reason)); + SmiTag(Signed(slot_id)), StringConstant(reason)); #endif // V8_TRACE_FEEDBACK_UPDATES } @@ -10285,14 +9876,16 @@ TNode<Map> CodeStubAssembler::LoadReceiverMap(SloppyTNode<Object> receiver) { [=] { return LoadMap(UncheckedCast<HeapObject>(receiver)); }); } -TNode<IntPtrT> CodeStubAssembler::TryToIntptr(Node* key, Label* miss) { +TNode<IntPtrT> CodeStubAssembler::TryToIntptr(SloppyTNode<Object> key, + Label* miss) { TVARIABLE(IntPtrT, var_intptr_key); Label done(this, &var_intptr_key), key_is_smi(this); GotoIf(TaggedIsSmi(key), &key_is_smi); + // Try to convert a heap number to a Smi. - GotoIfNot(IsHeapNumber(key), miss); + GotoIfNot(IsHeapNumber(CAST(key)), miss); { - TNode<Float64T> value = LoadHeapNumberValue(key); + TNode<Float64T> value = LoadHeapNumberValue(CAST(key)); TNode<Int32T> int_value = RoundFloat64ToInt32(value); GotoIfNot(Float64Equal(value, ChangeInt32ToFloat64(int_value)), miss); var_intptr_key = ChangeInt32ToIntPtr(int_value); @@ -10301,7 +9894,7 @@ TNode<IntPtrT> CodeStubAssembler::TryToIntptr(Node* key, Label* miss) { BIND(&key_is_smi); { - var_intptr_key = SmiUntag(key); + var_intptr_key = SmiUntag(CAST(key)); Goto(&done); } @@ -10354,7 +9947,7 @@ Node* CodeStubAssembler::EmitKeyedSloppyArguments( } Label if_mapped(this), if_unmapped(this), end(this, &var_result); TNode<IntPtrT> intptr_two = IntPtrConstant(2); - TNode<WordT> adjusted_length = IntPtrSub(elements_length, intptr_two); + TNode<IntPtrT> adjusted_length = IntPtrSub(elements_length, intptr_two); GotoIf(UintPtrGreaterThanOrEqual(key, adjusted_length), &if_unmapped); @@ -10510,33 +10103,35 @@ void CodeStubAssembler::StoreElement(Node* elements, ElementsKind kind, } } -Node* CodeStubAssembler::Int32ToUint8Clamped(Node* int32_value) { +TNode<Uint8T> CodeStubAssembler::Int32ToUint8Clamped( + TNode<Int32T> int32_value) { Label done(this); TNode<Int32T> int32_zero = Int32Constant(0); TNode<Int32T> int32_255 = Int32Constant(255); - VARIABLE(var_value, MachineRepresentation::kWord32, int32_value); + TVARIABLE(Word32T, var_value, int32_value); GotoIf(Uint32LessThanOrEqual(int32_value, int32_255), &done); - var_value.Bind(int32_zero); + var_value = int32_zero; GotoIf(Int32LessThan(int32_value, int32_zero), &done); - var_value.Bind(int32_255); + var_value = int32_255; Goto(&done); BIND(&done); - return var_value.value(); + return UncheckedCast<Uint8T>(var_value.value()); } -Node* CodeStubAssembler::Float64ToUint8Clamped(Node* float64_value) { +TNode<Uint8T> CodeStubAssembler::Float64ToUint8Clamped( + TNode<Float64T> float64_value) { Label done(this); - VARIABLE(var_value, MachineRepresentation::kWord32, Int32Constant(0)); + TVARIABLE(Word32T, var_value, Int32Constant(0)); GotoIf(Float64LessThanOrEqual(float64_value, Float64Constant(0.0)), &done); - var_value.Bind(Int32Constant(255)); + var_value = Int32Constant(255); GotoIf(Float64LessThanOrEqual(Float64Constant(255.0), float64_value), &done); { TNode<Float64T> rounded_value = Float64RoundToEven(float64_value); - var_value.Bind(TruncateFloat64ToWord32(rounded_value)); + var_value = TruncateFloat64ToWord32(rounded_value); Goto(&done); } BIND(&done); - return var_value.value(); + return UncheckedCast<Uint8T>(var_value.value()); } Node* CodeStubAssembler::PrepareValueForWriteToTypedArray( @@ -10716,8 +10311,8 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, GotoIfNot(UintPtrLessThan(intptr_key, length), &update_value_and_bailout); } - TNode<RawPtrT> backing_store = LoadJSTypedArrayBackingStore(CAST(object)); - StoreElement(backing_store, elements_kind, intptr_key, converted_value, + TNode<RawPtrT> data_ptr = LoadJSTypedArrayDataPtr(CAST(object)); + StoreElement(data_ptr, elements_kind, intptr_key, converted_value, parameter_mode); Goto(&done); @@ -10807,7 +10402,7 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, if (IsSmiElementsKind(elements_kind)) { GotoIfNot(TaggedIsSmi(value), bailout); } else if (IsDoubleElementsKind(elements_kind)) { - value = TryTaggedToFloat64(value, bailout); + value = TryTaggedToFloat64(CAST(value), bailout); } if (IsGrowStoreMode(store_mode) && @@ -11047,7 +10642,7 @@ TNode<IntPtrT> CodeStubAssembler::PageFromAddress(TNode<IntPtrT> address) { } TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector( - SloppyTNode<FeedbackVector> feedback_vector, TNode<Smi> slot) { + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot) { TNode<IntPtrT> size = IntPtrConstant(AllocationSite::kSizeWithWeakNext); TNode<HeapObject> site = Allocate(size, CodeStubAssembler::kPretenured); StoreMapNoWriteBarrier(site, RootIndex::kAllocationSiteWithWeakNextMap); @@ -11090,19 +10685,16 @@ TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector( StoreObjectField(site, AllocationSite::kWeakNextOffset, next_site); StoreFullTaggedNoWriteBarrier(site_list, site); - StoreFeedbackVectorSlot(feedback_vector, slot, site, UPDATE_WRITE_BARRIER, 0, - SMI_PARAMETERS); + StoreFeedbackVectorSlot(feedback_vector, slot, site); return CAST(site); } TNode<MaybeObject> CodeStubAssembler::StoreWeakReferenceInFeedbackVector( - SloppyTNode<FeedbackVector> feedback_vector, Node* slot, - SloppyTNode<HeapObject> value, int additional_offset, - ParameterMode parameter_mode) { + TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot, + TNode<HeapObject> value, int additional_offset) { TNode<MaybeObject> weak_value = MakeWeak(value); StoreFeedbackVectorSlot(feedback_vector, slot, weak_value, - UPDATE_WRITE_BARRIER, additional_offset, - parameter_mode); + UPDATE_WRITE_BARRIER, additional_offset); return weak_value; } @@ -11135,14 +10727,14 @@ TNode<Int32T> CodeStubAssembler::LoadElementsKind( return elements_kind; } -Node* CodeStubAssembler::BuildFastLoop( - const CodeStubAssembler::VariableList& vars, Node* start_index, - Node* end_index, const FastLoopBody& body, int increment, - ParameterMode parameter_mode, IndexAdvanceMode advance_mode) { - CSA_SLOW_ASSERT(this, MatchesParameterMode(start_index, parameter_mode)); - CSA_SLOW_ASSERT(this, MatchesParameterMode(end_index, parameter_mode)); - MachineRepresentation index_rep = ParameterRepresentation(parameter_mode); - VARIABLE(var, index_rep, start_index); +template <typename TIndex> +TNode<TIndex> CodeStubAssembler::BuildFastLoop(const VariableList& vars, + TNode<TIndex> start_index, + TNode<TIndex> end_index, + const FastLoopBody<TIndex>& body, + int increment, + IndexAdvanceMode advance_mode) { + TVARIABLE(TIndex, var, start_index); VariableList vars_copy(vars.begin(), vars.end(), zone()); vars_copy.push_back(&var); Label loop(this, vars_copy); @@ -11154,8 +10746,7 @@ Node* CodeStubAssembler::BuildFastLoop( // to force the loop header check at the end of the loop and branch forward to // it from the pre-header). The extra branch is slower in the case that the // loop actually iterates. - TNode<BoolT> first_check = - IntPtrOrSmiEqual(var.value(), end_index, parameter_mode); + TNode<BoolT> first_check = IntPtrOrSmiEqual(var.value(), end_index); int32_t first_check_val; if (ToInt32Constant(first_check, &first_check_val)) { if (first_check_val) return var.value(); @@ -11167,19 +10758,28 @@ Node* CodeStubAssembler::BuildFastLoop( BIND(&loop); { if (advance_mode == IndexAdvanceMode::kPre) { - Increment(&var, increment, parameter_mode); + Increment(&var, increment); } body(var.value()); if (advance_mode == IndexAdvanceMode::kPost) { - Increment(&var, increment, parameter_mode); + Increment(&var, increment); } - Branch(IntPtrOrSmiNotEqual(var.value(), end_index, parameter_mode), &loop, - &after_loop); + Branch(IntPtrOrSmiNotEqual(var.value(), end_index), &loop, &after_loop); } BIND(&after_loop); return var.value(); } +// Instantiate BuildFastLoop for Smi and IntPtrT. +template TNode<Smi> CodeStubAssembler::BuildFastLoop<Smi>( + const VariableList& vars, TNode<Smi> start_index, TNode<Smi> end_index, + const FastLoopBody<Smi>& body, int increment, + IndexAdvanceMode advance_mode); +template TNode<IntPtrT> CodeStubAssembler::BuildFastLoop<IntPtrT>( + const VariableList& vars, TNode<IntPtrT> start_index, + TNode<IntPtrT> end_index, const FastLoopBody<IntPtrT>& body, int increment, + IndexAdvanceMode advance_mode); + void CodeStubAssembler::BuildFastFixedArrayForEach( const CodeStubAssembler::VariableList& vars, Node* fixed_array, ElementsKind kind, Node* first_element_inclusive, @@ -11201,17 +10801,15 @@ void CodeStubAssembler::BuildFastFixedArrayForEach( if (direction == ForEachDirection::kForward) { for (int i = first_val; i < last_val; ++i) { TNode<IntPtrT> index = IntPtrConstant(i); - TNode<IntPtrT> offset = - ElementOffsetFromIndex(index, kind, INTPTR_PARAMETERS, - FixedArray::kHeaderSize - kHeapObjectTag); + TNode<IntPtrT> offset = ElementOffsetFromIndex( + index, kind, FixedArray::kHeaderSize - kHeapObjectTag); body(fixed_array, offset); } } else { for (int i = last_val - 1; i >= first_val; --i) { TNode<IntPtrT> index = IntPtrConstant(i); - TNode<IntPtrT> offset = - ElementOffsetFromIndex(index, kind, INTPTR_PARAMETERS, - FixedArray::kHeaderSize - kHeapObjectTag); + TNode<IntPtrT> offset = ElementOffsetFromIndex( + index, kind, FixedArray::kHeaderSize - kHeapObjectTag); body(fixed_array, offset); } } @@ -11228,11 +10826,10 @@ void CodeStubAssembler::BuildFastFixedArrayForEach( if (direction == ForEachDirection::kReverse) std::swap(start, limit); int increment = IsDoubleElementsKind(kind) ? kDoubleSize : kTaggedSize; - BuildFastLoop( + BuildFastLoop<IntPtrT>( vars, start, limit, - [fixed_array, &body](Node* offset) { body(fixed_array, offset); }, + [&](TNode<IntPtrT> offset) { body(fixed_array, offset); }, direction == ForEachDirection::kReverse ? -increment : increment, - INTPTR_PARAMETERS, direction == ForEachDirection::kReverse ? IndexAdvanceMode::kPre : IndexAdvanceMode::kPost); } @@ -11243,22 +10840,21 @@ void CodeStubAssembler::GotoIfFixedArraySizeDoesntFitInNewSpace( doesnt_fit); } -void CodeStubAssembler::InitializeFieldsWithRoot(Node* object, - Node* start_offset, - Node* end_offset, +void CodeStubAssembler::InitializeFieldsWithRoot(TNode<HeapObject> object, + TNode<IntPtrT> start_offset, + TNode<IntPtrT> end_offset, RootIndex root_index) { CSA_SLOW_ASSERT(this, TaggedIsNotSmi(object)); start_offset = IntPtrAdd(start_offset, IntPtrConstant(-kHeapObjectTag)); end_offset = IntPtrAdd(end_offset, IntPtrConstant(-kHeapObjectTag)); TNode<Object> root_value = LoadRoot(root_index); - BuildFastLoop( + BuildFastLoop<IntPtrT>( end_offset, start_offset, - [this, object, root_value](Node* current) { + [=](TNode<IntPtrT> current) { StoreNoWriteBarrier(MachineRepresentation::kTagged, object, current, root_value); }, - -kTaggedSize, INTPTR_PARAMETERS, - CodeStubAssembler::IndexAdvanceMode::kPre); + -kTaggedSize, CodeStubAssembler::IndexAdvanceMode::kPre); } void CodeStubAssembler::BranchIfNumberRelationalComparison( @@ -11384,11 +10980,9 @@ Operation Reverse(Operation op) { } } // anonymous namespace -Node* CodeStubAssembler::RelationalComparison(Operation op, - SloppyTNode<Object> left, - SloppyTNode<Object> right, - SloppyTNode<Context> context, - Variable* var_type_feedback) { +TNode<Oddball> CodeStubAssembler::RelationalComparison( + Operation op, TNode<Object> left, TNode<Object> right, + TNode<Context> context, TVariable<Smi>* var_type_feedback) { Label return_true(this), return_false(this), do_float_comparison(this), end(this); TVARIABLE(Oddball, var_result); // Actually only "true" or "false". @@ -11403,7 +10997,7 @@ Node* CodeStubAssembler::RelationalComparison(Operation op, if (var_type_feedback != nullptr) { // Initialize the type feedback to None. The current feedback is combined // with the previous feedback. - var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kNone)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kNone); loop_variable_list.push_back(var_type_feedback); } Label loop(this, loop_variable_list); @@ -11914,17 +11508,17 @@ void CodeStubAssembler::GenerateEqual_Same(SloppyTNode<Object> value, } // ES6 section 7.2.12 Abstract Equality Comparison -Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, - SloppyTNode<Object> right, - SloppyTNode<Context> context, - Variable* var_type_feedback) { +TNode<Oddball> CodeStubAssembler::Equal(SloppyTNode<Object> left, + SloppyTNode<Object> right, + SloppyTNode<Context> context, + TVariable<Smi>* var_type_feedback) { // This is a slightly optimized version of Object::Equals. Whenever you // change something functionality wise in here, remember to update the // Object::Equals method as well. Label if_equal(this), if_notequal(this), do_float_comparison(this), do_right_stringtonumber(this, Label::kDeferred), end(this); - VARIABLE(result, MachineRepresentation::kTagged); + TVARIABLE(Oddball, result); TVARIABLE(Float64T, var_left_float); TVARIABLE(Float64T, var_right_float); @@ -11984,7 +11578,7 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, GotoIf(IsHeapNumberMap(right_map), &if_right_heapnumber); // {left} is Smi and {right} is not HeapNumber or Smi. if (var_type_feedback != nullptr) { - var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } GotoIf(IsBooleanMap(right_map), &if_right_boolean); TNode<Uint16T> right_type = LoadMapInstanceType(right_map); @@ -12009,8 +11603,8 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, BIND(&if_right_bigint); { - result.Bind(CallRuntime(Runtime::kBigIntEqualToNumber, - NoContextConstant(), right, left)); + result = CAST(CallRuntime(Runtime::kBigIntEqualToNumber, + NoContextConstant(), right, left)); Goto(&end); } @@ -12046,7 +11640,8 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, BIND(&if_left_string); { GotoIfNot(IsStringInstanceType(right_type), &use_symmetry); - result.Bind(CallBuiltin(Builtins::kStringEqual, context, left, right)); + result = + CAST(CallBuiltin(Builtins::kStringEqual, context, left, right)); CombineFeedback(var_type_feedback, SmiOr(CollectFeedbackForString(left_type), CollectFeedbackForString(right_type))); @@ -12067,8 +11662,7 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, { Label if_right_boolean(this); if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } GotoIf(IsStringInstanceType(right_type), &do_right_stringtonumber); GotoIf(IsBooleanMap(right_map), &if_right_boolean); @@ -12098,38 +11692,35 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, BIND(&if_right_heapnumber); { if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } - result.Bind(CallRuntime(Runtime::kBigIntEqualToNumber, - NoContextConstant(), left, right)); + result = CAST(CallRuntime(Runtime::kBigIntEqualToNumber, + NoContextConstant(), left, right)); Goto(&end); } BIND(&if_right_bigint); { CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt); - result.Bind(CallRuntime(Runtime::kBigIntEqualToBigInt, - NoContextConstant(), left, right)); + result = CAST(CallRuntime(Runtime::kBigIntEqualToBigInt, + NoContextConstant(), left, right)); Goto(&end); } BIND(&if_right_string); { if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } - result.Bind(CallRuntime(Runtime::kBigIntEqualToString, - NoContextConstant(), left, right)); + result = CAST(CallRuntime(Runtime::kBigIntEqualToString, + NoContextConstant(), left, right)); Goto(&end); } BIND(&if_right_boolean); { if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } var_right = LoadObjectField(CAST(right), Oddball::kToNumberOffset); Goto(&loop); @@ -12154,8 +11745,8 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, if (var_type_feedback != nullptr) { // If {right} is undetectable, it must be either also // Null or Undefined, or a Receiver (aka document.all). - var_type_feedback->Bind(SmiConstant( - CompareOperationFeedback::kReceiverOrNullOrUndefined)); + *var_type_feedback = SmiConstant( + CompareOperationFeedback::kReceiverOrNullOrUndefined); } Goto(&if_equal); } @@ -12164,12 +11755,11 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, { if (var_type_feedback != nullptr) { // Track whether {right} is Null, Undefined or Receiver. - var_type_feedback->Bind(SmiConstant( - CompareOperationFeedback::kReceiverOrNullOrUndefined)); + *var_type_feedback = SmiConstant( + CompareOperationFeedback::kReceiverOrNullOrUndefined); GotoIf(IsJSReceiverInstanceType(right_type), &if_notequal); GotoIfNot(IsBooleanMap(right_map), &if_notequal); - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } Goto(&if_notequal); } @@ -12178,8 +11768,7 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, BIND(&if_left_boolean); { if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } // If {right} is a Boolean too, it must be a different Boolean. @@ -12200,7 +11789,7 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, if (var_type_feedback != nullptr) { Label if_right_symbol(this); GotoIf(IsSymbolInstanceType(right_type), &if_right_symbol); - var_type_feedback->Bind(SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); Goto(&if_notequal); BIND(&if_right_symbol); @@ -12218,8 +11807,7 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, // {left} is a Primitive and {right} is a JSReceiver, so swapping // the order is not observable. if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } Goto(&use_symmetry); } @@ -12254,8 +11842,8 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, // When we get here, {right} must be either Null or Undefined. CSA_ASSERT(this, IsNullOrUndefined(right)); if (var_type_feedback != nullptr) { - var_type_feedback->Bind(SmiConstant( - CompareOperationFeedback::kReceiverOrNullOrUndefined)); + *var_type_feedback = SmiConstant( + CompareOperationFeedback::kReceiverOrNullOrUndefined); } Branch(IsUndetectableMap(left_map), &if_equal, &if_notequal); } @@ -12265,8 +11853,7 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, // {right} is a Primitive, and neither Null or Undefined; // convert {left} to Primitive too. if (var_type_feedback != nullptr) { - var_type_feedback->Bind( - SmiConstant(CompareOperationFeedback::kAny)); + *var_type_feedback = SmiConstant(CompareOperationFeedback::kAny); } Callable callable = CodeFactory::NonPrimitiveToPrimitive(isolate()); var_left = CallStub(callable, context, left); @@ -12298,13 +11885,13 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, BIND(&if_equal); { - result.Bind(TrueConstant()); + result = TrueConstant(); Goto(&end); } BIND(&if_notequal); { - result.Bind(FalseConstant()); + result = FalseConstant(); Goto(&end); } @@ -12312,9 +11899,9 @@ Node* CodeStubAssembler::Equal(SloppyTNode<Object> left, return result.value(); } -TNode<Oddball> CodeStubAssembler::StrictEqual(SloppyTNode<Object> lhs, - SloppyTNode<Object> rhs, - Variable* var_type_feedback) { +TNode<Oddball> CodeStubAssembler::StrictEqual( + SloppyTNode<Object> lhs, SloppyTNode<Object> rhs, + TVariable<Smi>* var_type_feedback) { // Pseudo-code for the algorithm below: // // if (lhs == rhs) { @@ -12482,7 +12069,7 @@ TNode<Oddball> CodeStubAssembler::StrictEqual(SloppyTNode<Object> lhs, CollectFeedbackForString(lhs_instance_type); TNode<Smi> rhs_feedback = CollectFeedbackForString(rhs_instance_type); - var_type_feedback->Bind(SmiOr(lhs_feedback, rhs_feedback)); + *var_type_feedback = SmiOr(lhs_feedback, rhs_feedback); } result = CAST(CallBuiltin(Builtins::kStringEqual, NoContextConstant(), lhs, rhs)); @@ -12556,7 +12143,7 @@ TNode<Oddball> CodeStubAssembler::StrictEqual(SloppyTNode<Object> lhs, BIND(&if_lhsisoddball); { - STATIC_ASSERT(LAST_PRIMITIVE_TYPE == ODDBALL_TYPE); + STATIC_ASSERT(LAST_PRIMITIVE_HEAP_OBJECT_TYPE == ODDBALL_TYPE); GotoIf(IsBooleanMap(rhs_map), &if_not_equivalent_types); GotoIf(Int32LessThan(rhs_instance_type, Int32Constant(ODDBALL_TYPE)), @@ -12855,8 +12442,8 @@ TNode<Oddball> CodeStubAssembler::HasProperty(SloppyTNode<Context> context, return result.value(); } -Node* CodeStubAssembler::Typeof(Node* value) { - VARIABLE(result_var, MachineRepresentation::kTagged); +TNode<String> CodeStubAssembler::Typeof(SloppyTNode<Object> value) { + TVARIABLE(String, result_var); Label return_number(this, Label::kDeferred), if_oddball(this), return_function(this), return_undefined(this), return_object(this), @@ -12864,7 +12451,8 @@ Node* CodeStubAssembler::Typeof(Node* value) { GotoIf(TaggedIsSmi(value), &return_number); - TNode<Map> map = LoadMap(value); + TNode<HeapObject> value_heap_object = CAST(value); + TNode<Map> map = LoadMap(value_heap_object); GotoIf(IsHeapNumberMap(map), &return_number); @@ -12890,49 +12478,50 @@ Node* CodeStubAssembler::Typeof(Node* value) { GotoIf(IsBigIntInstanceType(instance_type), &return_bigint); CSA_ASSERT(this, InstanceTypeEqual(instance_type, SYMBOL_TYPE)); - result_var.Bind(HeapConstant(isolate()->factory()->symbol_string())); + result_var = HeapConstant(isolate()->factory()->symbol_string()); Goto(&return_result); BIND(&return_number); { - result_var.Bind(HeapConstant(isolate()->factory()->number_string())); + result_var = HeapConstant(isolate()->factory()->number_string()); Goto(&return_result); } BIND(&if_oddball); { - TNode<Object> type = LoadObjectField(value, Oddball::kTypeOfOffset); - result_var.Bind(type); + TNode<String> type = + CAST(LoadObjectField(value_heap_object, Oddball::kTypeOfOffset)); + result_var = type; Goto(&return_result); } BIND(&return_function); { - result_var.Bind(HeapConstant(isolate()->factory()->function_string())); + result_var = HeapConstant(isolate()->factory()->function_string()); Goto(&return_result); } BIND(&return_undefined); { - result_var.Bind(HeapConstant(isolate()->factory()->undefined_string())); + result_var = HeapConstant(isolate()->factory()->undefined_string()); Goto(&return_result); } BIND(&return_object); { - result_var.Bind(HeapConstant(isolate()->factory()->object_string())); + result_var = HeapConstant(isolate()->factory()->object_string()); Goto(&return_result); } BIND(&return_string); { - result_var.Bind(HeapConstant(isolate()->factory()->string_string())); + result_var = HeapConstant(isolate()->factory()->string_string()); Goto(&return_result); } BIND(&return_bigint); { - result_var.Bind(HeapConstant(isolate()->factory()->bigint_string())); + result_var = HeapConstant(isolate()->factory()->bigint_string()); Goto(&return_result); } @@ -12941,7 +12530,7 @@ Node* CodeStubAssembler::Typeof(Node* value) { } TNode<Object> CodeStubAssembler::GetSuperConstructor( - SloppyTNode<Context> context, SloppyTNode<JSFunction> active_function) { + TNode<Context> context, TNode<JSFunction> active_function) { Label is_not_constructor(this, Label::kDeferred), out(this); TVARIABLE(Object, result); @@ -13004,9 +12593,10 @@ TNode<JSReceiver> CodeStubAssembler::SpeciesConstructor( return var_result.value(); } -Node* CodeStubAssembler::InstanceOf(Node* object, Node* callable, - Node* context) { - VARIABLE(var_result, MachineRepresentation::kTagged); +TNode<Oddball> CodeStubAssembler::InstanceOf(TNode<Object> object, + TNode<Object> callable, + TNode<Context> context) { + TVARIABLE(Oddball, var_result); Label if_notcallable(this, Label::kDeferred), if_notreceiver(this, Label::kDeferred), if_otherhandler(this), if_nohandler(this, Label::kDeferred), return_true(this), @@ -13014,7 +12604,7 @@ Node* CodeStubAssembler::InstanceOf(Node* object, Node* callable, // Ensure that the {callable} is actually a JSReceiver. GotoIf(TaggedIsSmi(callable), &if_notreceiver); - GotoIfNot(IsJSReceiver(callable), &if_notreceiver); + GotoIfNot(IsJSReceiver(CAST(callable)), &if_notreceiver); // Load the @@hasInstance property from {callable}. TNode<Object> inst_of_handler = @@ -13032,8 +12622,8 @@ Node* CodeStubAssembler::InstanceOf(Node* object, Node* callable, // Call to Function.prototype[@@hasInstance] directly. Callable builtin(BUILTIN_CODE(isolate(), FunctionPrototypeHasInstance), CallTrampolineDescriptor{}); - Node* result = CallJS(builtin, context, inst_of_handler, callable, object); - var_result.Bind(result); + var_result = + CAST(CallJS(builtin, context, inst_of_handler, callable, object)); Goto(&return_result); } @@ -13055,12 +12645,11 @@ Node* CodeStubAssembler::InstanceOf(Node* object, Node* callable, BIND(&if_nohandler); { // Ensure that the {callable} is actually Callable. - GotoIfNot(IsCallable(callable), &if_notcallable); + GotoIfNot(IsCallable(CAST(callable)), &if_notcallable); // Use the OrdinaryHasInstance algorithm. - TNode<Object> result = - CallBuiltin(Builtins::kOrdinaryHasInstance, context, callable, object); - var_result.Bind(result); + var_result = CAST( + CallBuiltin(Builtins::kOrdinaryHasInstance, context, callable, object)); Goto(&return_result); } @@ -13071,11 +12660,11 @@ Node* CodeStubAssembler::InstanceOf(Node* object, Node* callable, { ThrowTypeError(context, MessageTemplate::kNonObjectInInstanceOfCheck); } BIND(&return_true); - var_result.Bind(TrueConstant()); + var_result = TrueConstant(); Goto(&return_result); BIND(&return_false); - var_result.Bind(FalseConstant()); + var_result = FalseConstant(); Goto(&return_result); BIND(&return_result); @@ -13294,9 +12883,8 @@ TNode<JSObject> CodeStubAssembler::AllocateJSIteratorResult( return CAST(result); } -Node* CodeStubAssembler::AllocateJSIteratorResultForEntry(Node* context, - Node* key, - Node* value) { +TNode<JSObject> CodeStubAssembler::AllocateJSIteratorResultForEntry( + TNode<Context> context, TNode<Object> key, SloppyTNode<Object> value) { TNode<NativeContext> native_context = LoadNativeContext(context); TNode<Smi> length = SmiConstant(2); int const elements_size = FixedArray::SizeFor(2); @@ -13326,7 +12914,7 @@ Node* CodeStubAssembler::AllocateJSIteratorResultForEntry(Node* context, StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kValueOffset, array); StoreObjectFieldRoot(result, JSIteratorResult::kDoneOffset, RootIndex::kFalseValue); - return result; + return CAST(result); } TNode<JSReceiver> CodeStubAssembler::ArraySpeciesCreate(TNode<Context> context, @@ -13393,21 +12981,19 @@ TNode<UintPtrT> CodeStubAssembler::LoadJSTypedArrayLength( return LoadObjectField<UintPtrT>(typed_array, JSTypedArray::kLengthOffset); } -CodeStubArguments::CodeStubArguments( - CodeStubAssembler* assembler, Node* argc, Node* fp, - CodeStubAssembler::ParameterMode param_mode, ReceiverMode receiver_mode) +CodeStubArguments::CodeStubArguments(CodeStubAssembler* assembler, + TNode<IntPtrT> argc, TNode<RawPtrT> fp, + ReceiverMode receiver_mode) : assembler_(assembler), - argc_mode_(param_mode), receiver_mode_(receiver_mode), argc_(argc), base_(), fp_(fp != nullptr ? fp : assembler_->LoadFramePointer()) { TNode<IntPtrT> offset = assembler_->ElementOffsetFromIndex( - argc_, SYSTEM_POINTER_ELEMENTS, param_mode, + argc_, SYSTEM_POINTER_ELEMENTS, (StandardFrameConstants::kFixedSlotCountAboveFp - 1) * kSystemPointerSize); - base_ = - assembler_->UncheckedCast<RawPtrT>(assembler_->IntPtrAdd(fp_, offset)); + base_ = assembler_->RawPtrAdd(fp_, offset); } TNode<Object> CodeStubArguments::GetReceiver() const { @@ -13422,24 +13008,18 @@ void CodeStubArguments::SetReceiver(TNode<Object> object) const { base_, assembler_->IntPtrConstant(kSystemPointerSize), object); } -TNode<WordT> CodeStubArguments::AtIndexPtr( - Node* index, CodeStubAssembler::ParameterMode mode) const { - using Node = compiler::Node; - Node* negated_index = assembler_->IntPtrOrSmiSub( - assembler_->IntPtrOrSmiConstant(0, mode), index, mode); +TNode<RawPtrT> CodeStubArguments::AtIndexPtr(TNode<IntPtrT> index) const { + TNode<IntPtrT> negated_index = + assembler_->IntPtrOrSmiSub(assembler_->IntPtrConstant(0), index); TNode<IntPtrT> offset = assembler_->ElementOffsetFromIndex( - negated_index, SYSTEM_POINTER_ELEMENTS, mode, 0); - return assembler_->IntPtrAdd(assembler_->UncheckedCast<IntPtrT>(base_), - offset); + negated_index, SYSTEM_POINTER_ELEMENTS, 0); + return assembler_->RawPtrAdd(base_, offset); } -TNode<Object> CodeStubArguments::AtIndex( - Node* index, CodeStubAssembler::ParameterMode mode) const { - DCHECK_EQ(argc_mode_, mode); - CSA_ASSERT(assembler_, - assembler_->UintPtrOrSmiLessThan(index, GetLength(mode), mode)); +TNode<Object> CodeStubArguments::AtIndex(TNode<IntPtrT> index) const { + CSA_ASSERT(assembler_, assembler_->UintPtrOrSmiLessThan(index, GetLength())); return assembler_->UncheckedCast<Object>( - assembler_->LoadFullTagged(AtIndexPtr(index, mode))); + assembler_->LoadFullTagged(AtIndexPtr(index))); } TNode<Object> CodeStubArguments::AtIndex(int index) const { @@ -13452,9 +13032,8 @@ TNode<Object> CodeStubArguments::GetOptionalArgumentValue( CodeStubAssembler::Label argument_missing(assembler_), argument_done(assembler_, &result); - assembler_->GotoIf(assembler_->UintPtrOrSmiGreaterThanOrEqual( - assembler_->IntPtrOrSmiConstant(index, argc_mode_), - argc_, argc_mode_), + assembler_->GotoIf(assembler_->UintPtrGreaterThanOrEqual( + assembler_->IntPtrConstant(index), argc_), &argument_missing); result = AtIndex(index); assembler_->Goto(&argument_done); @@ -13473,10 +13052,8 @@ TNode<Object> CodeStubArguments::GetOptionalArgumentValue( CodeStubAssembler::Label argument_missing(assembler_), argument_done(assembler_, &result); - assembler_->GotoIf( - assembler_->UintPtrOrSmiGreaterThanOrEqual( - assembler_->IntPtrToParameter(index, argc_mode_), argc_, argc_mode_), - &argument_missing); + assembler_->GotoIf(assembler_->UintPtrGreaterThanOrEqual(index, argc_), + &argument_missing); result = AtIndex(index); assembler_->Goto(&argument_done); @@ -13490,43 +13067,38 @@ TNode<Object> CodeStubArguments::GetOptionalArgumentValue( void CodeStubArguments::ForEach( const CodeStubAssembler::VariableList& vars, - const CodeStubArguments::ForEachBodyFunction& body, Node* first, Node* last, - CodeStubAssembler::ParameterMode mode) { + const CodeStubArguments::ForEachBodyFunction& body, TNode<IntPtrT> first, + TNode<IntPtrT> last) const { assembler_->Comment("CodeStubArguments::ForEach"); if (first == nullptr) { - first = assembler_->IntPtrOrSmiConstant(0, mode); + first = assembler_->IntPtrConstant(0); } if (last == nullptr) { - DCHECK_EQ(mode, argc_mode_); last = argc_; } - TNode<IntPtrT> start = assembler_->IntPtrSub( - assembler_->UncheckedCast<IntPtrT>(base_), - assembler_->ElementOffsetFromIndex(first, SYSTEM_POINTER_ELEMENTS, mode)); - TNode<IntPtrT> end = assembler_->IntPtrSub( - assembler_->UncheckedCast<IntPtrT>(base_), - assembler_->ElementOffsetFromIndex(last, SYSTEM_POINTER_ELEMENTS, mode)); - assembler_->BuildFastLoop( + TNode<RawPtrT> start = assembler_->RawPtrSub( + base_, + assembler_->ElementOffsetFromIndex(first, SYSTEM_POINTER_ELEMENTS)); + TNode<RawPtrT> end = assembler_->RawPtrSub( + base_, assembler_->ElementOffsetFromIndex(last, SYSTEM_POINTER_ELEMENTS)); + assembler_->BuildFastLoop<RawPtrT>( vars, start, end, - [this, &body](Node* current) { - Node* arg = assembler_->Load(MachineType::AnyTagged(), current); + [&](TNode<RawPtrT> current) { + TNode<Object> arg = assembler_->Load<Object>(current); body(arg); }, - -kSystemPointerSize, CodeStubAssembler::INTPTR_PARAMETERS, - CodeStubAssembler::IndexAdvanceMode::kPost); + -kSystemPointerSize, CodeStubAssembler::IndexAdvanceMode::kPost); } void CodeStubArguments::PopAndReturn(Node* value) { - Node* pop_count; + TNode<IntPtrT> pop_count; if (receiver_mode_ == ReceiverMode::kHasReceiver) { - pop_count = assembler_->IntPtrOrSmiAdd( - argc_, assembler_->IntPtrOrSmiConstant(1, argc_mode_), argc_mode_); + pop_count = assembler_->IntPtrAdd(argc_, assembler_->IntPtrConstant(1)); } else { pop_count = argc_; } - assembler_->PopAndReturn(assembler_->ParameterToIntPtr(pop_count, argc_mode_), - value); + assembler_->PopAndReturn(pop_count, value); } TNode<BoolT> CodeStubAssembler::IsFastElementsKind( @@ -13642,21 +13214,15 @@ Node* CodeStubAssembler:: } TNode<Code> CodeStubAssembler::LoadBuiltin(TNode<Smi> builtin_id) { - CSA_ASSERT(this, SmiGreaterThanOrEqual(builtin_id, SmiConstant(0))); - CSA_ASSERT(this, - SmiLessThan(builtin_id, SmiConstant(Builtins::builtin_count))); + CSA_ASSERT(this, SmiBelow(builtin_id, SmiConstant(Builtins::builtin_count))); - int const kSmiShiftBits = kSmiShiftSize + kSmiTagSize; - int index_shift = kSystemPointerSizeLog2 - kSmiShiftBits; - TNode<WordT> table_index = - index_shift >= 0 - ? WordShl(BitcastTaggedSignedToWord(builtin_id), index_shift) - : WordSar(BitcastTaggedSignedToWord(builtin_id), -index_shift); - - return CAST( - Load(MachineType::TaggedPointer(), + TNode<IntPtrT> offset = + ElementOffsetFromIndex(SmiToBInt(builtin_id), SYSTEM_POINTER_ELEMENTS); + + return CAST(BitcastWordToTagged( + Load(MachineType::Pointer(), ExternalConstant(ExternalReference::builtins_address(isolate())), - table_index)); + offset))); } TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( @@ -13765,11 +13331,9 @@ TNode<Code> CodeStubAssembler::GetSharedFunctionInfoCode( return sfi_code.value(); } -Node* CodeStubAssembler::AllocateFunctionWithMapAndContext(Node* map, - Node* shared_info, - Node* context) { - CSA_SLOW_ASSERT(this, IsMap(map)); - +TNode<JSFunction> CodeStubAssembler::AllocateFunctionWithMapAndContext( + TNode<Map> map, TNode<SharedFunctionInfo> shared_info, + TNode<Context> context) { TNode<Code> const code = GetSharedFunctionInfoCode(shared_info); // TODO(ishell): All the callers of this function pass map loaded from @@ -13790,7 +13354,7 @@ Node* CodeStubAssembler::AllocateFunctionWithMapAndContext(Node* map, shared_info); StoreObjectFieldNoWriteBarrier(fun, JSFunction::kContextOffset, context); StoreObjectFieldNoWriteBarrier(fun, JSFunction::kCodeOffset, code); - return fun; + return CAST(fun); } void CodeStubAssembler::CheckPrototypeEnumCache(Node* receiver, @@ -13839,8 +13403,9 @@ void CodeStubAssembler::CheckPrototypeEnumCache(Node* receiver, } } -Node* CodeStubAssembler::CheckEnumCache(Node* receiver, Label* if_empty, - Label* if_runtime) { +TNode<Map> CodeStubAssembler::CheckEnumCache(TNode<HeapObject> receiver, + Label* if_empty, + Label* if_runtime) { Label if_fast(this), if_cache(this), if_no_cache(this, Label::kDeferred); TNode<Map> receiver_map = LoadMap(receiver); @@ -13855,7 +13420,7 @@ Node* CodeStubAssembler::CheckEnumCache(Node* receiver, Label* if_empty, { // Avoid runtime-call for empty dictionary receivers. GotoIfNot(IsDictionaryMap(receiver_map), if_runtime); - TNode<NameDictionary> properties = CAST(LoadSlowProperties(receiver)); + TNode<NameDictionary> properties = CAST(LoadSlowProperties(CAST(receiver))); TNode<Smi> length = GetNumberOfElements(properties); GotoIfNot(TaggedEqual(length, SmiConstant(0)), if_runtime); // Check that there are no elements on the {receiver} and its prototype @@ -13881,8 +13446,7 @@ TNode<Object> CodeStubAssembler::GetArgumentValue(TorqueStructArguments args, TorqueStructArguments CodeStubAssembler::GetFrameArguments( TNode<RawPtrT> frame, TNode<IntPtrT> argc) { - return CodeStubArguments(this, argc, frame, INTPTR_PARAMETERS) - .GetTorqueArguments(); + return CodeStubArguments(this, argc, frame).GetTorqueArguments(); } void CodeStubAssembler::Print(const char* s) { @@ -13976,9 +13540,8 @@ TNode<JSArray> CodeStubAssembler::ArrayCreate(TNode<Context> context, // TODO(delphick): Consider using // AllocateUninitializedJSArrayWithElements to avoid initializing an // array and then writing over it. - array = - AllocateJSArray(PACKED_SMI_ELEMENTS, array_map, length, SmiConstant(0), - nullptr, ParameterMode::SMI_PARAMETERS); + array = AllocateJSArray(PACKED_SMI_ELEMENTS, array_map, length, + SmiConstant(0), {}, ParameterMode::SMI_PARAMETERS); Goto(&done); BIND(&done); |