diff options
Diffstat (limited to 'deps/v8/src/codegen/code-stub-assembler.cc')
| -rw-r--r-- | deps/v8/src/codegen/code-stub-assembler.cc | 871 |
1 files changed, 476 insertions, 395 deletions
diff --git a/deps/v8/src/codegen/code-stub-assembler.cc b/deps/v8/src/codegen/code-stub-assembler.cc index d967d84874..390746c27d 100644 --- a/deps/v8/src/codegen/code-stub-assembler.cc +++ b/deps/v8/src/codegen/code-stub-assembler.cc @@ -63,57 +63,27 @@ void CodeStubAssembler::HandleBreakOnNode() { void CodeStubAssembler::Assert(const BranchGenerator& branch, const char* message, const char* file, int line, - Node* extra_node1, const char* extra_node1_name, - Node* extra_node2, const char* extra_node2_name, - Node* extra_node3, const char* extra_node3_name, - Node* extra_node4, const char* extra_node4_name, - Node* extra_node5, - const char* extra_node5_name) { + std::initializer_list<ExtraNode> extra_nodes) { #if defined(DEBUG) if (FLAG_debug_code) { - Check(branch, message, file, line, extra_node1, extra_node1_name, - extra_node2, extra_node2_name, extra_node3, extra_node3_name, - extra_node4, extra_node4_name, extra_node5, extra_node5_name); + Check(branch, message, file, line, extra_nodes); } #endif } void CodeStubAssembler::Assert(const NodeGenerator& condition_body, const char* message, const char* file, int line, - Node* extra_node1, const char* extra_node1_name, - Node* extra_node2, const char* extra_node2_name, - Node* extra_node3, const char* extra_node3_name, - Node* extra_node4, const char* extra_node4_name, - Node* extra_node5, - const char* extra_node5_name) { + std::initializer_list<ExtraNode> extra_nodes) { #if defined(DEBUG) if (FLAG_debug_code) { - Check(condition_body, message, file, line, extra_node1, extra_node1_name, - extra_node2, extra_node2_name, extra_node3, extra_node3_name, - extra_node4, extra_node4_name, extra_node5, extra_node5_name); + Check(condition_body, message, file, line, extra_nodes); } #endif } -#ifdef DEBUG -namespace { -void MaybePrintNodeWithName(CodeStubAssembler* csa, Node* node, - const char* node_name) { - if (node != nullptr) { - csa->CallRuntime(Runtime::kPrintWithNameForAssert, csa->SmiConstant(0), - csa->StringConstant(node_name), node); - } -} -} // namespace -#endif - void CodeStubAssembler::Check(const BranchGenerator& branch, const char* message, const char* file, int line, - Node* extra_node1, const char* extra_node1_name, - Node* extra_node2, const char* extra_node2_name, - Node* extra_node3, const char* extra_node3_name, - Node* extra_node4, const char* extra_node4_name, - Node* extra_node5, const char* extra_node5_name) { + std::initializer_list<ExtraNode> extra_nodes) { Label ok(this); Label not_ok(this, Label::kDeferred); if (message != nullptr && FLAG_code_comments) { @@ -124,9 +94,7 @@ void CodeStubAssembler::Check(const BranchGenerator& branch, branch(&ok, ¬_ok); BIND(¬_ok); - FailAssert(message, file, line, extra_node1, extra_node1_name, extra_node2, - extra_node2_name, extra_node3, extra_node3_name, extra_node4, - extra_node4_name, extra_node5, extra_node5_name); + FailAssert(message, file, line, extra_nodes); BIND(&ok); Comment("] Assert"); @@ -134,20 +102,14 @@ void CodeStubAssembler::Check(const BranchGenerator& branch, void CodeStubAssembler::Check(const NodeGenerator& condition_body, const char* message, const char* file, int line, - Node* extra_node1, const char* extra_node1_name, - Node* extra_node2, const char* extra_node2_name, - Node* extra_node3, const char* extra_node3_name, - Node* extra_node4, const char* extra_node4_name, - Node* extra_node5, const char* extra_node5_name) { + std::initializer_list<ExtraNode> extra_nodes) { BranchGenerator branch = [=](Label* ok, Label* not_ok) { Node* condition = condition_body(); DCHECK_NOT_NULL(condition); Branch(condition, ok, not_ok); }; - Check(branch, message, file, line, extra_node1, extra_node1_name, extra_node2, - extra_node2_name, extra_node3, extra_node3_name, extra_node4, - extra_node4_name, extra_node5, extra_node5_name); + Check(branch, message, file, line, extra_nodes); } void CodeStubAssembler::FastCheck(TNode<BoolT> condition) { @@ -162,31 +124,25 @@ void CodeStubAssembler::FastCheck(TNode<BoolT> condition) { } void CodeStubAssembler::FailAssert( - const char* message, const char* file, int line, Node* extra_node1, - const char* extra_node1_name, Node* extra_node2, - const char* extra_node2_name, Node* extra_node3, - const char* extra_node3_name, Node* extra_node4, - const char* extra_node4_name, Node* extra_node5, - const char* extra_node5_name) { + const char* message, const char* file, int line, + std::initializer_list<ExtraNode> extra_nodes) { DCHECK_NOT_NULL(message); EmbeddedVector<char, 1024> chars; if (file != nullptr) { - SNPrintF(chars, "CSA_ASSERT failed: %s [%s:%d]\n", message, file, line); - } else { - SNPrintF(chars, "CSA_ASSERT failed: %s\n", message); + SNPrintF(chars, "%s [%s:%d]", message, file, line); + message = chars.begin(); } - Node* message_node = StringConstant(chars.begin()); + Node* message_node = StringConstant(message); #ifdef DEBUG // Only print the extra nodes in debug builds. - MaybePrintNodeWithName(this, extra_node1, extra_node1_name); - MaybePrintNodeWithName(this, extra_node2, extra_node2_name); - MaybePrintNodeWithName(this, extra_node3, extra_node3_name); - MaybePrintNodeWithName(this, extra_node4, extra_node4_name); - MaybePrintNodeWithName(this, extra_node5, extra_node5_name); + for (auto& node : extra_nodes) { + CallRuntime(Runtime::kPrintWithNameForAssert, SmiConstant(0), + StringConstant(node.second), node.first); + } #endif - DebugAbort(message_node); + AbortCSAAssert(message_node); Unreachable(); } @@ -567,7 +523,7 @@ TNode<Float64T> CodeStubAssembler::Float64Trunc(SloppyTNode<Float64T> x) { TNode<BoolT> CodeStubAssembler::IsValidSmi(TNode<Smi> smi) { if (SmiValuesAre31Bits() && kSystemPointerSize == kInt64Size) { // Check that the Smi value is properly sign-extended. - TNode<IntPtrT> value = Signed(BitcastTaggedToWord(smi)); + TNode<IntPtrT> value = Signed(BitcastTaggedSignedToWord(smi)); return WordEqual(value, ChangeInt32ToIntPtr(TruncateIntPtrToInt32(value))); } return Int32TrueConstant(); @@ -611,7 +567,8 @@ TNode<IntPtrT> CodeStubAssembler::SmiUntag(SloppyTNode<Smi> value) { if (ToIntPtrConstant(value, constant_value)) { return IntPtrConstant(constant_value >> (kSmiShiftSize + kSmiTagSize)); } - return Signed(WordSar(BitcastTaggedToWord(value), SmiShiftBitsConstant())); + return Signed( + WordSar(BitcastTaggedSignedToWord(value), SmiShiftBitsConstant())); } TNode<Int32T> CodeStubAssembler::SmiToInt32(SloppyTNode<Smi> value) { @@ -660,13 +617,14 @@ TNode<Int32T> CodeStubAssembler::TryInt32Mul(TNode<Int32T> a, TNode<Int32T> b, TNode<Smi> CodeStubAssembler::TrySmiAdd(TNode<Smi> lhs, TNode<Smi> rhs, Label* if_overflow) { if (SmiValuesAre32Bits()) { - return BitcastWordToTaggedSigned(TryIntPtrAdd( - BitcastTaggedToWord(lhs), BitcastTaggedToWord(rhs), if_overflow)); + return BitcastWordToTaggedSigned( + TryIntPtrAdd(BitcastTaggedSignedToWord(lhs), + BitcastTaggedSignedToWord(rhs), if_overflow)); } else { DCHECK(SmiValuesAre31Bits()); - TNode<PairT<Int32T, BoolT>> pair = - Int32AddWithOverflow(TruncateIntPtrToInt32(BitcastTaggedToWord(lhs)), - TruncateIntPtrToInt32(BitcastTaggedToWord(rhs))); + TNode<PairT<Int32T, BoolT>> pair = Int32AddWithOverflow( + TruncateIntPtrToInt32(BitcastTaggedSignedToWord(lhs)), + TruncateIntPtrToInt32(BitcastTaggedSignedToWord(rhs))); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, if_overflow); TNode<Int32T> result = Projection<0>(pair); @@ -678,16 +636,16 @@ TNode<Smi> CodeStubAssembler::TrySmiSub(TNode<Smi> lhs, TNode<Smi> rhs, Label* if_overflow) { if (SmiValuesAre32Bits()) { TNode<PairT<IntPtrT, BoolT>> pair = IntPtrSubWithOverflow( - BitcastTaggedToWord(lhs), BitcastTaggedToWord(rhs)); + BitcastTaggedSignedToWord(lhs), BitcastTaggedSignedToWord(rhs)); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, if_overflow); TNode<IntPtrT> result = Projection<0>(pair); return BitcastWordToTaggedSigned(result); } else { DCHECK(SmiValuesAre31Bits()); - TNode<PairT<Int32T, BoolT>> pair = - Int32SubWithOverflow(TruncateIntPtrToInt32(BitcastTaggedToWord(lhs)), - TruncateIntPtrToInt32(BitcastTaggedToWord(rhs))); + TNode<PairT<Int32T, BoolT>> pair = Int32SubWithOverflow( + TruncateIntPtrToInt32(BitcastTaggedSignedToWord(lhs)), + TruncateIntPtrToInt32(BitcastTaggedSignedToWord(rhs))); TNode<BoolT> overflow = Projection<1>(pair); GotoIf(overflow, if_overflow); TNode<Int32T> result = Projection<0>(pair); @@ -933,7 +891,7 @@ TNode<Smi> CodeStubAssembler::TrySmiDiv(TNode<Smi> dividend, TNode<Smi> divisor, BIND(&divisor_is_not_minus_one); TNode<Int32T> untagged_result = Int32Div(untagged_dividend, untagged_divisor); - TNode<Int32T> truncated = Signed(Int32Mul(untagged_result, untagged_divisor)); + TNode<Int32T> truncated = Int32Mul(untagged_result, untagged_divisor); // Do floating point division if the remainder is not 0. GotoIf(Word32NotEqual(untagged_dividend, truncated), bailout); @@ -973,9 +931,12 @@ TNode<BoolT> CodeStubAssembler::TaggedIsSmi(TNode<MaybeObject> a) { } TNode<BoolT> CodeStubAssembler::TaggedIsNotSmi(SloppyTNode<Object> a) { - return WordNotEqual( - WordAnd(BitcastTaggedToWord(a), IntPtrConstant(kSmiTagMask)), - IntPtrConstant(0)); + // Although BitcastTaggedSignedToWord is generally unsafe on HeapObjects, we + // can nonetheless use it to inspect the Smi tag. The assumption here is that + // the GC will not exchange Smis for HeapObjects or vice-versa. + TNode<IntPtrT> a_bitcast = BitcastTaggedSignedToWord(UncheckedCast<Smi>(a)); + return WordNotEqual(WordAnd(a_bitcast, IntPtrConstant(kSmiTagMask)), + IntPtrConstant(0)); } TNode<BoolT> CodeStubAssembler::TaggedIsPositiveSmi(SloppyTNode<Object> a) { @@ -1031,7 +992,7 @@ void CodeStubAssembler::BranchIfPrototypesHaveNoElements( TNode<Int32T> prototype_instance_type = LoadMapInstanceType(prototype_map); // Pessimistically assume elements if a Proxy, Special API Object, - // or JSValue wrapper is found on the prototype chain. After this + // or JSPrimitiveWrapper wrapper is found on the prototype chain. After this // instance type check, it's not necessary to check for interceptors or // access checks. Label if_custom(this, Label::kDeferred), if_notcustom(this); @@ -1040,11 +1001,12 @@ void CodeStubAssembler::BranchIfPrototypesHaveNoElements( BIND(&if_custom); { - // For string JSValue wrappers we still support the checks as long - // as they wrap the empty string. - GotoIfNot(InstanceTypeEqual(prototype_instance_type, JS_VALUE_TYPE), - possibly_elements); - Node* prototype_value = LoadJSValueValue(prototype); + // For string JSPrimitiveWrapper wrappers we still support the checks as + // long as they wrap the empty string. + GotoIfNot( + InstanceTypeEqual(prototype_instance_type, JS_PRIMITIVE_WRAPPER_TYPE), + possibly_elements); + Node* prototype_value = LoadJSPrimitiveWrapperValue(prototype); Branch(IsEmptyString(prototype_value), &if_notcustom, possibly_elements); } @@ -1121,20 +1083,23 @@ TNode<HeapObject> CodeStubAssembler::AllocateRaw(TNode<IntPtrT> size_in_bytes, Label runtime_call(this, Label::kDeferred), no_runtime_call(this), out(this); bool needs_double_alignment = flags & kDoubleAlignment; + bool allow_large_object_allocation = flags & kAllowLargeObjectAllocation; - if (flags & kAllowLargeObjectAllocation) { + if (allow_large_object_allocation) { Label next(this); GotoIf(IsRegularHeapObjectSize(size_in_bytes), &next); + TNode<Smi> runtime_flags = SmiConstant(Smi::FromInt( + AllocateDoubleAlignFlag::encode(needs_double_alignment) | + AllowLargeObjectAllocationFlag::encode(allow_large_object_allocation))); if (FLAG_young_generation_large_objects) { - result = CallRuntime(Runtime::kAllocateInYoungGeneration, - NoContextConstant(), SmiTag(size_in_bytes)); + result = + CallRuntime(Runtime::kAllocateInYoungGeneration, NoContextConstant(), + SmiTag(size_in_bytes), runtime_flags); } else { - TNode<Smi> alignment_flag = SmiConstant(Smi::FromInt( - AllocateDoubleAlignFlag::encode(needs_double_alignment))); result = CallRuntime(Runtime::kAllocateInOldGeneration, NoContextConstant(), - SmiTag(size_in_bytes), alignment_flag); + SmiTag(size_in_bytes), runtime_flags); } Goto(&out); @@ -1161,15 +1126,17 @@ TNode<HeapObject> CodeStubAssembler::AllocateRaw(TNode<IntPtrT> size_in_bytes, BIND(&runtime_call); { + TNode<Smi> runtime_flags = SmiConstant(Smi::FromInt( + AllocateDoubleAlignFlag::encode(needs_double_alignment) | + AllowLargeObjectAllocationFlag::encode(allow_large_object_allocation))); if (flags & kPretenured) { - TNode<Smi> runtime_flags = SmiConstant(Smi::FromInt( - AllocateDoubleAlignFlag::encode(needs_double_alignment))); result = CallRuntime(Runtime::kAllocateInOldGeneration, NoContextConstant(), SmiTag(size_in_bytes), runtime_flags); } else { - result = CallRuntime(Runtime::kAllocateInYoungGeneration, - NoContextConstant(), SmiTag(size_in_bytes)); + result = + CallRuntime(Runtime::kAllocateInYoungGeneration, NoContextConstant(), + SmiTag(size_in_bytes), runtime_flags); } Goto(&out); } @@ -1394,14 +1361,15 @@ Node* CodeStubAssembler::LoadBufferObject(Node* buffer, int offset, Node* CodeStubAssembler::LoadObjectField(SloppyTNode<HeapObject> object, int offset, MachineType type) { CSA_ASSERT(this, IsStrong(object)); - return Load(type, object, IntPtrConstant(offset - kHeapObjectTag)); + return LoadFromObject(type, object, IntPtrConstant(offset - kHeapObjectTag)); } Node* CodeStubAssembler::LoadObjectField(SloppyTNode<HeapObject> object, SloppyTNode<IntPtrT> offset, MachineType type) { CSA_ASSERT(this, IsStrong(object)); - return Load(type, object, IntPtrSub(offset, IntPtrConstant(kHeapObjectTag))); + return LoadFromObject(type, object, + IntPtrSub(offset, IntPtrConstant(kHeapObjectTag))); } TNode<IntPtrT> CodeStubAssembler::LoadAndUntagObjectField( @@ -1469,12 +1437,18 @@ TNode<Float64T> CodeStubAssembler::LoadHeapNumberValue( object, HeapNumber::kValueOffset, MachineType::Float64())); } +TNode<Map> CodeStubAssembler::GetStructMap(InstanceType instance_type) { + Handle<Map> map_handle(Map::GetStructMap(isolate(), instance_type), + isolate()); + return HeapConstant(map_handle); +} + TNode<Map> CodeStubAssembler::LoadMap(SloppyTNode<HeapObject> object) { return UncheckedCast<Map>(LoadObjectField(object, HeapObject::kMapOffset, MachineType::TaggedPointer())); } -TNode<Int32T> CodeStubAssembler::LoadInstanceType( +TNode<Uint16T> CodeStubAssembler::LoadInstanceType( SloppyTNode<HeapObject> object) { return LoadMapInstanceType(LoadMap(object)); } @@ -1591,8 +1565,8 @@ TNode<Uint32T> CodeStubAssembler::LoadMapBitField3(SloppyTNode<Map> map) { LoadObjectField(map, Map::kBitField3Offset, MachineType::Uint32())); } -TNode<Int32T> CodeStubAssembler::LoadMapInstanceType(SloppyTNode<Map> map) { - return UncheckedCast<Int32T>( +TNode<Uint16T> CodeStubAssembler::LoadMapInstanceType(SloppyTNode<Map> map) { + return UncheckedCast<Uint16T>( LoadObjectField(map, Map::kInstanceTypeOffset, MachineType::Uint16())); } @@ -1700,12 +1674,10 @@ TNode<Object> CodeStubAssembler::LoadMapBackPointer(SloppyTNode<Map> map) { TNode<Uint32T> CodeStubAssembler::EnsureOnlyHasSimpleProperties( TNode<Map> map, TNode<Int32T> instance_type, Label* bailout) { - // This check can have false positives, since it applies to any JSValueType. + // This check can have false positives, since it applies to any + // JSPrimitiveWrapper type. GotoIf(IsCustomElementsReceiverInstanceType(instance_type), bailout); - GotoIf(IsSetWord32(LoadMapBitField2(map), Map::HasHiddenPrototypeBit::kMask), - bailout); - TNode<Uint32T> bit_field3 = LoadMapBitField3(map); GotoIf(IsSetWord32(bit_field3, Map::IsDictionaryMapBit::kMask), bailout); @@ -1810,9 +1782,9 @@ Node* CodeStubAssembler::PointerToSeqStringData(Node* seq_string) { IntPtrConstant(SeqOneByteString::kHeaderSize - kHeapObjectTag)); } -Node* CodeStubAssembler::LoadJSValueValue(Node* object) { - CSA_ASSERT(this, IsJSValue(object)); - return LoadObjectField(object, JSValue::kValueOffset); +Node* CodeStubAssembler::LoadJSPrimitiveWrapperValue(Node* object) { + CSA_ASSERT(this, IsJSPrimitiveWrapper(object)); + return LoadObjectField(object, JSPrimitiveWrapper::kValueOffset); } void CodeStubAssembler::DispatchMaybeObject(TNode<MaybeObject> maybe_object, @@ -1941,11 +1913,13 @@ TNode<IntPtrT> CodeStubAssembler::LoadArrayLength( return LoadAndUntagWeakFixedArrayLength(array); } -template <typename Array> -TNode<MaybeObject> CodeStubAssembler::LoadArrayElement( - TNode<Array> array, int array_header_size, Node* index_node, - int additional_offset, ParameterMode parameter_mode, - LoadSensitivity needs_poisoning) { +template <typename Array, typename T> +TNode<T> CodeStubAssembler::LoadArrayElement(TNode<Array> array, + int array_header_size, + Node* index_node, + int additional_offset, + ParameterMode parameter_mode, + LoadSensitivity needs_poisoning) { CSA_ASSERT(this, IntPtrGreaterThanOrEqual( ParameterToIntPtr(index_node, parameter_mode), IntPtrConstant(0))); @@ -1955,8 +1929,13 @@ TNode<MaybeObject> CodeStubAssembler::LoadArrayElement( parameter_mode, header_size); CSA_ASSERT(this, IsOffsetInBounds(offset, LoadArrayLength(array), array_header_size)); - return UncheckedCast<MaybeObject>( - Load(MachineType::AnyTagged(), array, offset, needs_poisoning)); + constexpr MachineType machine_type = MachineTypeOf<T>::value; + // TODO(gsps): Remove the Load case once LoadFromObject supports poisoning + if (needs_poisoning == LoadSensitivity::kSafe) { + return UncheckedCast<T>(LoadFromObject(machine_type, array, offset)); + } else { + return UncheckedCast<T>(Load(machine_type, array, offset, needs_poisoning)); + } } template TNode<MaybeObject> @@ -2046,8 +2025,8 @@ TNode<RawPtrT> CodeStubAssembler::LoadJSTypedArrayBackingStore( IntPtrAdd(external_pointer, BitcastTaggedToWord(base_pointer))); } -Node* CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged( - Node* data_pointer, Node* offset) { +TNode<BigInt> CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged( + SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset) { if (Is64()) { TNode<IntPtrT> value = UncheckedCast<IntPtrT>( Load(MachineType::IntPtr(), data_pointer, offset)); @@ -2059,13 +2038,15 @@ Node* CodeStubAssembler::LoadFixedBigInt64ArrayElementAsTagged( Load(MachineType::UintPtr(), data_pointer, offset)); TNode<IntPtrT> low = UncheckedCast<IntPtrT>( Load(MachineType::UintPtr(), data_pointer, - Int32Add(offset, Int32Constant(kSystemPointerSize)))); + Int32Add(TruncateIntPtrToInt32(offset), + Int32Constant(kSystemPointerSize)))); #else TNode<IntPtrT> low = UncheckedCast<IntPtrT>( Load(MachineType::UintPtr(), data_pointer, offset)); TNode<IntPtrT> high = UncheckedCast<IntPtrT>( Load(MachineType::UintPtr(), data_pointer, - Int32Add(offset, Int32Constant(kSystemPointerSize)))); + Int32Add(TruncateIntPtrToInt32(offset), + Int32Constant(kSystemPointerSize)))); #endif return BigIntFromInt32Pair(low, high); } @@ -2176,8 +2157,9 @@ TNode<BigInt> CodeStubAssembler::BigIntFromInt64(TNode<IntPtrT> value) { return var_result.value(); } -Node* CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged( - Node* data_pointer, Node* offset) { +compiler::TNode<BigInt> +CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged( + SloppyTNode<RawPtrT> data_pointer, SloppyTNode<IntPtrT> offset) { Label if_zero(this), done(this); if (Is64()) { TNode<UintPtrT> value = UncheckedCast<UintPtrT>( @@ -2190,13 +2172,15 @@ Node* CodeStubAssembler::LoadFixedBigUint64ArrayElementAsTagged( Load(MachineType::UintPtr(), data_pointer, offset)); TNode<UintPtrT> low = UncheckedCast<UintPtrT>( Load(MachineType::UintPtr(), data_pointer, - Int32Add(offset, Int32Constant(kSystemPointerSize)))); + Int32Add(TruncateIntPtrToInt32(offset), + Int32Constant(kSystemPointerSize)))); #else TNode<UintPtrT> low = UncheckedCast<UintPtrT>( Load(MachineType::UintPtr(), data_pointer, offset)); TNode<UintPtrT> high = UncheckedCast<UintPtrT>( Load(MachineType::UintPtr(), data_pointer, - Int32Add(offset, Int32Constant(kSystemPointerSize)))); + Int32Add(TruncateIntPtrToInt32(offset), + Int32Constant(kSystemPointerSize)))); #endif return BigIntFromUint32Pair(low, high); } @@ -2244,10 +2228,10 @@ TNode<BigInt> CodeStubAssembler::BigIntFromUint64(TNode<UintPtrT> value) { return var_result.value(); } -Node* CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( - Node* data_pointer, Node* index_node, ElementsKind elements_kind, +TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( + TNode<RawPtrT> data_pointer, Node* index_node, ElementsKind elements_kind, ParameterMode parameter_mode) { - Node* offset = + TNode<IntPtrT> offset = ElementOffsetFromIndex(index_node, elements_kind, parameter_mode, 0); switch (elements_kind) { case UINT8_ELEMENTS: /* fall through */ @@ -2281,7 +2265,8 @@ Node* CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( } TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( - TNode<WordT> data_pointer, TNode<Smi> index, TNode<Int32T> elements_kind) { + TNode<RawPtrT> data_pointer, TNode<Smi> index, + TNode<Int32T> elements_kind) { TVARIABLE(Numeric, var_result); Label done(this), if_unknown_type(this, Label::kDeferred); int32_t elements_kinds[] = { @@ -2307,12 +2292,12 @@ TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( BIND(&if_unknown_type); Unreachable(); -#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) \ - BIND(&if_##type##array); \ - { \ - var_result = CAST(LoadFixedTypedArrayElementAsTagged( \ - data_pointer, index, TYPE##_ELEMENTS, SMI_PARAMETERS)); \ - Goto(&done); \ +#define TYPED_ARRAY_CASE(Type, type, TYPE, ctype) \ + BIND(&if_##type##array); \ + { \ + var_result = LoadFixedTypedArrayElementAsTagged( \ + data_pointer, index, TYPE##_ELEMENTS, SMI_PARAMETERS); \ + Goto(&done); \ } TYPED_ARRAYS(TYPED_ARRAY_CASE) #undef TYPED_ARRAY_CASE @@ -2323,8 +2308,7 @@ TNode<Numeric> CodeStubAssembler::LoadFixedTypedArrayElementAsTagged( void CodeStubAssembler::StoreJSTypedArrayElementFromTagged( TNode<Context> context, TNode<JSTypedArray> typed_array, - TNode<Object> index_node, TNode<Object> value, ElementsKind elements_kind, - ParameterMode parameter_mode) { + TNode<Smi> index_node, TNode<Object> value, ElementsKind elements_kind) { TNode<RawPtrT> data_pointer = LoadJSTypedArrayBackingStore(typed_array); switch (elements_kind) { case UINT8_ELEMENTS: @@ -2333,26 +2317,26 @@ void CodeStubAssembler::StoreJSTypedArrayElementFromTagged( case UINT16_ELEMENTS: case INT16_ELEMENTS: StoreElement(data_pointer, elements_kind, index_node, - SmiToInt32(CAST(value)), parameter_mode); + SmiToInt32(CAST(value)), SMI_PARAMETERS); break; case UINT32_ELEMENTS: case INT32_ELEMENTS: StoreElement(data_pointer, elements_kind, index_node, - TruncateTaggedToWord32(context, value), parameter_mode); + TruncateTaggedToWord32(context, value), SMI_PARAMETERS); break; case FLOAT32_ELEMENTS: StoreElement(data_pointer, elements_kind, index_node, TruncateFloat64ToFloat32(LoadHeapNumberValue(CAST(value))), - parameter_mode); + SMI_PARAMETERS); break; case FLOAT64_ELEMENTS: StoreElement(data_pointer, elements_kind, index_node, - LoadHeapNumberValue(CAST(value)), parameter_mode); + LoadHeapNumberValue(CAST(value)), SMI_PARAMETERS); break; case BIGUINT64_ELEMENTS: case BIGINT64_ELEMENTS: StoreElement(data_pointer, elements_kind, index_node, - UncheckedCast<BigInt>(value), parameter_mode); + UncheckedCast<BigInt>(value), SMI_PARAMETERS); break; default: UNREACHABLE(); @@ -2925,15 +2909,12 @@ TNode<Int32T> CodeStubAssembler::EnsureArrayPushable(TNode<Map> map, // Disallow pushing onto prototypes. It might be the JSArray prototype. // Disallow pushing onto non-extensible objects. Comment("Disallow pushing onto prototypes"); - Node* bit_field2 = LoadMapBitField2(map); - int mask = Map::IsPrototypeMapBit::kMask | Map::IsExtensibleBit::kMask; - Node* test = Word32And(bit_field2, Int32Constant(mask)); - GotoIf(Word32NotEqual(test, Int32Constant(Map::IsExtensibleBit::kMask)), - bailout); + GotoIfNot(IsExtensibleNonPrototypeMap(map), bailout); EnsureArrayLengthWritable(map, bailout); - TNode<Uint32T> kind = DecodeWord32<Map::ElementsKindBits>(bit_field2); + TNode<Uint32T> kind = + DecodeWord32<Map::ElementsKindBits>(LoadMapBitField2(map)); return Signed(kind); } @@ -3022,7 +3003,7 @@ void CodeStubAssembler::TryStoreArrayElement(ElementsKind kind, GotoIfNotNumber(value, bailout); } if (IsDoubleElementsKind(kind)) { - value = ChangeNumberToFloat64(value); + value = ChangeNumberToFloat64(CAST(value)); } StoreElement(elements, kind, index, value, mode); } @@ -3131,14 +3112,10 @@ TNode<BigInt> CodeStubAssembler::AllocateBigInt(TNode<IntPtrT> length) { } TNode<BigInt> CodeStubAssembler::AllocateRawBigInt(TNode<IntPtrT> length) { - // This is currently used only for 64-bit wide BigInts. If more general - // applicability is required, a large-object check must be added. - CSA_ASSERT(this, UintPtrLessThan(length, IntPtrConstant(3))); - TNode<IntPtrT> size = IntPtrAdd(IntPtrConstant(BigInt::kHeaderSize), Signed(WordShl(length, kSystemPointerSizeLog2))); - Node* raw_result = Allocate(size, kNone); + Node* raw_result = Allocate(size, kAllowLargeObjectAllocation); StoreMapNoWriteBarrier(raw_result, RootIndex::kBigIntMap); if (FIELD_SIZE(BigInt::kOptionalPaddingOffset) != 0) { DCHECK_EQ(4, FIELD_SIZE(BigInt::kOptionalPaddingOffset)); @@ -3155,11 +3132,26 @@ void CodeStubAssembler::StoreBigIntBitfield(TNode<BigInt> bigint, MachineRepresentation::kWord32); } -void CodeStubAssembler::StoreBigIntDigit(TNode<BigInt> bigint, int digit_index, +void CodeStubAssembler::StoreBigIntDigit(TNode<BigInt> bigint, + intptr_t digit_index, TNode<UintPtrT> digit) { + CHECK_LE(0, digit_index); + CHECK_LT(digit_index, BigInt::kMaxLength); StoreObjectFieldNoWriteBarrier( - bigint, BigInt::kDigitsOffset + digit_index * kSystemPointerSize, digit, - UintPtrT::kMachineRepresentation); + bigint, + BigInt::kDigitsOffset + + static_cast<int>(digit_index) * kSystemPointerSize, + digit, UintPtrT::kMachineRepresentation); +} + +void CodeStubAssembler::StoreBigIntDigit(TNode<BigInt> bigint, + TNode<IntPtrT> digit_index, + TNode<UintPtrT> digit) { + TNode<IntPtrT> offset = + IntPtrAdd(IntPtrConstant(BigInt::kDigitsOffset), + IntPtrMul(digit_index, IntPtrConstant(kSystemPointerSize))); + StoreObjectFieldNoWriteBarrier(bigint, offset, digit, + UintPtrT::kMachineRepresentation); } TNode<Word32T> CodeStubAssembler::LoadBigIntBitfield(TNode<BigInt> bigint) { @@ -3168,10 +3160,23 @@ TNode<Word32T> CodeStubAssembler::LoadBigIntBitfield(TNode<BigInt> bigint) { } TNode<UintPtrT> CodeStubAssembler::LoadBigIntDigit(TNode<BigInt> bigint, - int digit_index) { - return UncheckedCast<UintPtrT>(LoadObjectField( - bigint, BigInt::kDigitsOffset + digit_index * kSystemPointerSize, - MachineType::UintPtr())); + intptr_t digit_index) { + CHECK_LE(0, digit_index); + CHECK_LT(digit_index, BigInt::kMaxLength); + return UncheckedCast<UintPtrT>( + LoadObjectField(bigint, + BigInt::kDigitsOffset + + static_cast<int>(digit_index) * kSystemPointerSize, + MachineType::UintPtr())); +} + +TNode<UintPtrT> CodeStubAssembler::LoadBigIntDigit(TNode<BigInt> bigint, + TNode<IntPtrT> digit_index) { + TNode<IntPtrT> offset = + IntPtrAdd(IntPtrConstant(BigInt::kDigitsOffset), + IntPtrMul(digit_index, IntPtrConstant(kSystemPointerSize))); + return UncheckedCast<UintPtrT>( + LoadObjectField(bigint, offset, MachineType::UintPtr())); } TNode<ByteArray> CodeStubAssembler::AllocateByteArray(TNode<UintPtrT> length, @@ -3440,16 +3445,16 @@ TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary( } TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionary( - TNode<IntPtrT> at_least_space_for) { + TNode<IntPtrT> at_least_space_for, AllocationFlags flags) { CSA_ASSERT(this, UintPtrLessThanOrEqual( at_least_space_for, IntPtrConstant(NameDictionary::kMaxCapacity))); TNode<IntPtrT> capacity = HashTableComputeCapacity(at_least_space_for); - return AllocateNameDictionaryWithCapacity(capacity); + return AllocateNameDictionaryWithCapacity(capacity, flags); } TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionaryWithCapacity( - TNode<IntPtrT> capacity) { + TNode<IntPtrT> capacity, AllocationFlags flags) { CSA_ASSERT(this, WordIsPowerOfTwo(capacity)); CSA_ASSERT(this, IntPtrGreaterThan(capacity, IntPtrConstant(0))); TNode<IntPtrT> length = EntryToIndex<NameDictionary>(capacity); @@ -3457,39 +3462,51 @@ TNode<NameDictionary> CodeStubAssembler::AllocateNameDictionaryWithCapacity( TimesTaggedSize(length), IntPtrConstant(NameDictionary::kHeaderSize)); TNode<NameDictionary> result = - UncheckedCast<NameDictionary>(AllocateInNewSpace(store_size)); - Comment("Initialize NameDictionary"); + UncheckedCast<NameDictionary>(Allocate(store_size, flags)); + // Initialize FixedArray fields. - DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kNameDictionaryMap)); - StoreMapNoWriteBarrier(result, RootIndex::kNameDictionaryMap); - StoreObjectFieldNoWriteBarrier(result, FixedArray::kLengthOffset, - SmiFromIntPtr(length)); + { + DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kNameDictionaryMap)); + StoreMapNoWriteBarrier(result, RootIndex::kNameDictionaryMap); + StoreObjectFieldNoWriteBarrier(result, FixedArray::kLengthOffset, + SmiFromIntPtr(length)); + } + // Initialized HashTable fields. - TNode<Smi> zero = SmiConstant(0); - StoreFixedArrayElement(result, NameDictionary::kNumberOfElementsIndex, zero, - SKIP_WRITE_BARRIER); - StoreFixedArrayElement(result, NameDictionary::kNumberOfDeletedElementsIndex, - zero, SKIP_WRITE_BARRIER); - StoreFixedArrayElement(result, NameDictionary::kCapacityIndex, - SmiTag(capacity), SKIP_WRITE_BARRIER); - // Initialize Dictionary fields. - TNode<HeapObject> filler = UndefinedConstant(); - StoreFixedArrayElement(result, NameDictionary::kNextEnumerationIndexIndex, - SmiConstant(PropertyDetails::kInitialIndex), - SKIP_WRITE_BARRIER); - StoreFixedArrayElement(result, NameDictionary::kObjectHashIndex, - SmiConstant(PropertyArray::kNoHashSentinel), - SKIP_WRITE_BARRIER); + { + TNode<Smi> zero = SmiConstant(0); + StoreFixedArrayElement(result, NameDictionary::kNumberOfElementsIndex, zero, + SKIP_WRITE_BARRIER); + StoreFixedArrayElement(result, + NameDictionary::kNumberOfDeletedElementsIndex, zero, + SKIP_WRITE_BARRIER); + StoreFixedArrayElement(result, NameDictionary::kCapacityIndex, + SmiTag(capacity), SKIP_WRITE_BARRIER); + // Initialize Dictionary fields. + StoreFixedArrayElement(result, NameDictionary::kNextEnumerationIndexIndex, + SmiConstant(PropertyDetails::kInitialIndex), + SKIP_WRITE_BARRIER); + StoreFixedArrayElement(result, NameDictionary::kObjectHashIndex, + SmiConstant(PropertyArray::kNoHashSentinel), + SKIP_WRITE_BARRIER); + } // Initialize NameDictionary elements. - TNode<WordT> result_word = BitcastTaggedToWord(result); - TNode<WordT> start_address = IntPtrAdd( - result_word, IntPtrConstant(NameDictionary::OffsetOfElementAt( - NameDictionary::kElementsStartIndex) - - kHeapObjectTag)); - TNode<WordT> end_address = IntPtrAdd( - result_word, IntPtrSub(store_size, IntPtrConstant(kHeapObjectTag))); - StoreFieldsNoWriteBarrier(start_address, end_address, filler); + { + TNode<WordT> result_word = BitcastTaggedToWord(result); + TNode<WordT> start_address = IntPtrAdd( + result_word, IntPtrConstant(NameDictionary::OffsetOfElementAt( + NameDictionary::kElementsStartIndex) - + kHeapObjectTag)); + TNode<WordT> end_address = IntPtrAdd( + result_word, IntPtrSub(store_size, IntPtrConstant(kHeapObjectTag))); + + TNode<HeapObject> filler = UndefinedConstant(); + DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kUndefinedValue)); + + StoreFieldsNoWriteBarrier(start_address, end_address, filler); + } + return result; } @@ -3605,6 +3622,17 @@ TNode<CollectionType> CodeStubAssembler::AllocateSmallOrderedHashTable( StoreMapNoWriteBarrier(table_obj, small_ordered_hash_map); TNode<CollectionType> table = UncheckedCast<CollectionType>(table_obj); + { + // This store overlaps with the header fields stored below. + // Since it happens first, it effectively still just zero-initializes the + // padding. + constexpr int offset = + RoundDown<kTaggedSize>(CollectionType::PaddingOffset()); + STATIC_ASSERT(offset + kTaggedSize == CollectionType::PaddingOffset() + + CollectionType::PaddingSize()); + StoreObjectFieldNoWriteBarrier(table, offset, SmiConstant(0)); + } + // Initialize the SmallOrderedHashTable fields. StoreObjectByteNoWriteBarrier( table, CollectionType::NumberOfBucketsOffset(), @@ -3748,8 +3776,9 @@ void CodeStubAssembler::InitializeStructBody(Node* object, Node* map, StoreFieldsNoWriteBarrier(start_address, end_address, filler); } -Node* CodeStubAssembler::AllocateJSObjectFromMap( - Node* map, Node* properties, Node* elements, AllocationFlags flags, +TNode<JSObject> CodeStubAssembler::AllocateJSObjectFromMap( + SloppyTNode<Map> map, SloppyTNode<HeapObject> properties, + SloppyTNode<FixedArray> elements, AllocationFlags flags, SlackTrackingMode slack_tracking_mode) { CSA_ASSERT(this, IsMap(map)); CSA_ASSERT(this, Word32BinaryNot(IsJSFunctionMap(map))); @@ -3761,7 +3790,7 @@ Node* CodeStubAssembler::AllocateJSObjectFromMap( StoreMapNoWriteBarrier(object, map); InitializeJSObjectFromMap(object, map, instance_size, properties, elements, slack_tracking_mode); - return object; + return CAST(object); } void CodeStubAssembler::InitializeJSObjectFromMap( @@ -5508,7 +5537,7 @@ void CodeStubAssembler::TaggedToWord32OrBigIntImpl( } BIND(&is_heap_number); - var_word32->Bind(TruncateHeapNumberValueToWord32(value)); + var_word32->Bind(TruncateHeapNumberValueToWord32(CAST(value))); CombineFeedback(var_feedback, BinaryOperationFeedback::kNumber); Goto(if_number); @@ -5521,9 +5550,10 @@ void CodeStubAssembler::TaggedToWord32OrBigIntImpl( } } -Node* CodeStubAssembler::TruncateHeapNumberValueToWord32(Node* object) { +TNode<Int32T> CodeStubAssembler::TruncateHeapNumberValueToWord32( + TNode<HeapNumber> object) { Node* value = LoadHeapNumberValue(object); - return TruncateFloat64ToWord32(value); + return Signed(TruncateFloat64ToWord32(value)); } void CodeStubAssembler::TryHeapNumberToSmi(TNode<HeapNumber> number, @@ -5731,10 +5761,7 @@ TNode<Uint32T> CodeStubAssembler::ChangeNumberToUint32(TNode<Number> value) { return var_result.value(); } -TNode<Float64T> CodeStubAssembler::ChangeNumberToFloat64( - SloppyTNode<Number> value) { - // TODO(tebbi): Remove assert once argument is TNode instead of SloppyTNode. - CSA_SLOW_ASSERT(this, IsNumber(value)); +TNode<Float64T> CodeStubAssembler::ChangeNumberToFloat64(TNode<Number> value) { TVARIABLE(Float64T, result); Label smi(this); Label done(this, &result); @@ -5795,43 +5822,43 @@ TNode<WordT> CodeStubAssembler::TimesDoubleSize(SloppyTNode<WordT> value) { return WordShl(value, kDoubleSizeLog2); } -Node* CodeStubAssembler::ToThisValue(Node* context, Node* value, - PrimitiveType primitive_type, - char const* method_name) { - // We might need to loop once due to JSValue unboxing. - VARIABLE(var_value, MachineRepresentation::kTagged, value); +TNode<Object> CodeStubAssembler::ToThisValue(TNode<Context> context, + TNode<Object> value, + PrimitiveType primitive_type, + char const* method_name) { + // We might need to loop once due to JSPrimitiveWrapper unboxing. + TVARIABLE(Object, var_value, value); Label loop(this, &var_value), done_loop(this), done_throw(this, Label::kDeferred); Goto(&loop); BIND(&loop); { - // Load the current {value}. - value = var_value.value(); - // Check if the {value} is a Smi or a HeapObject. - GotoIf(TaggedIsSmi(value), (primitive_type == PrimitiveType::kNumber) - ? &done_loop - : &done_throw); + GotoIf( + TaggedIsSmi(var_value.value()), + (primitive_type == PrimitiveType::kNumber) ? &done_loop : &done_throw); + + TNode<HeapObject> value = CAST(var_value.value()); // Load the map of the {value}. - Node* value_map = LoadMap(value); + TNode<Map> value_map = LoadMap(value); // Load the instance type of the {value}. - Node* value_instance_type = LoadMapInstanceType(value_map); + TNode<Uint16T> value_instance_type = LoadMapInstanceType(value_map); - // Check if {value} is a JSValue. - Label if_valueisvalue(this, Label::kDeferred), if_valueisnotvalue(this); - Branch(InstanceTypeEqual(value_instance_type, JS_VALUE_TYPE), - &if_valueisvalue, &if_valueisnotvalue); + // Check if {value} is a JSPrimitiveWrapper. + Label if_valueiswrapper(this, Label::kDeferred), if_valueisnotwrapper(this); + Branch(InstanceTypeEqual(value_instance_type, JS_PRIMITIVE_WRAPPER_TYPE), + &if_valueiswrapper, &if_valueisnotwrapper); - BIND(&if_valueisvalue); + BIND(&if_valueiswrapper); { // Load the actual value from the {value}. - var_value.Bind(LoadObjectField(value, JSValue::kValueOffset)); + var_value = LoadObjectField(value, JSPrimitiveWrapper::kValueOffset); Goto(&loop); } - BIND(&if_valueisnotvalue); + BIND(&if_valueisnotwrapper); { switch (primitive_type) { case PrimitiveType::kBoolean: @@ -5988,13 +6015,12 @@ TNode<BoolT> CodeStubAssembler::InstanceTypeEqual( TNode<BoolT> CodeStubAssembler::IsDictionaryMap(SloppyTNode<Map> map) { CSA_SLOW_ASSERT(this, IsMap(map)); - Node* bit_field3 = LoadMapBitField3(map); - return IsSetWord32<Map::IsDictionaryMapBit>(bit_field3); + return IsSetWord32<Map::IsDictionaryMapBit>(LoadMapBitField3(map)); } TNode<BoolT> CodeStubAssembler::IsExtensibleMap(SloppyTNode<Map> map) { CSA_ASSERT(this, IsMap(map)); - return IsSetWord32<Map::IsExtensibleBit>(LoadMapBitField2(map)); + return IsSetWord32<Map::IsExtensibleBit>(LoadMapBitField3(map)); } TNode<BoolT> CodeStubAssembler::IsFrozenOrSealedElementsKindMap( @@ -6007,7 +6033,7 @@ TNode<BoolT> CodeStubAssembler::IsFrozenOrSealedElementsKindMap( TNode<BoolT> CodeStubAssembler::IsExtensibleNonPrototypeMap(TNode<Map> map) { int kMask = Map::IsExtensibleBit::kMask | Map::IsPrototypeMapBit::kMask; int kExpected = Map::IsExtensibleBit::kMask; - return Word32Equal(Word32And(LoadMapBitField2(map), Int32Constant(kMask)), + return Word32Equal(Word32And(LoadMapBitField3(map), Int32Constant(kMask)), Int32Constant(kExpected)); } @@ -6072,10 +6098,13 @@ TNode<BoolT> CodeStubAssembler::IsTypedArraySpeciesProtectorCellInvalid() { return WordEqual(cell_value, invalid); } -TNode<BoolT> CodeStubAssembler::IsRegExpSpeciesProtectorCellInvalid() { - Node* invalid = SmiConstant(Isolate::kProtectorInvalid); - Node* cell = LoadRoot(RootIndex::kRegExpSpeciesProtector); - Node* cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); +TNode<BoolT> CodeStubAssembler::IsRegExpSpeciesProtectorCellInvalid( + TNode<Context> native_context) { + CSA_ASSERT(this, IsNativeContext(native_context)); + TNode<PropertyCell> cell = CAST(LoadContextElement( + native_context, Context::REGEXP_SPECIES_PROTECTOR_INDEX)); + TNode<Object> cell_value = LoadObjectField(cell, PropertyCell::kValueOffset); + TNode<Smi> invalid = SmiConstant(Isolate::kProtectorInvalid); return WordEqual(cell_value, invalid); } @@ -6270,6 +6299,15 @@ TNode<BoolT> CodeStubAssembler::IsJSGlobalProxyInstanceType( return InstanceTypeEqual(instance_type, JS_GLOBAL_PROXY_TYPE); } +TNode<BoolT> CodeStubAssembler::IsJSGlobalProxyMap(SloppyTNode<Map> map) { + return IsJSGlobalProxyInstanceType(LoadMapInstanceType(map)); +} + +TNode<BoolT> CodeStubAssembler::IsJSGlobalProxy( + SloppyTNode<HeapObject> object) { + return IsJSGlobalProxyMap(LoadMap(object)); +} + TNode<BoolT> CodeStubAssembler::IsJSObjectInstanceType( SloppyTNode<Int32T> instance_type) { STATIC_ASSERT(LAST_JS_OBJECT_TYPE == LAST_TYPE); @@ -6304,26 +6342,22 @@ TNode<BoolT> CodeStubAssembler::IsJSStringIterator( return HasInstanceType(object, JS_STRING_ITERATOR_TYPE); } -TNode<BoolT> CodeStubAssembler::IsJSGlobalProxy( - SloppyTNode<HeapObject> object) { - return HasInstanceType(object, JS_GLOBAL_PROXY_TYPE); -} - TNode<BoolT> CodeStubAssembler::IsMap(SloppyTNode<HeapObject> map) { return IsMetaMap(LoadMap(map)); } -TNode<BoolT> CodeStubAssembler::IsJSValueInstanceType( +TNode<BoolT> CodeStubAssembler::IsJSPrimitiveWrapperInstanceType( SloppyTNode<Int32T> instance_type) { - return InstanceTypeEqual(instance_type, JS_VALUE_TYPE); + return InstanceTypeEqual(instance_type, JS_PRIMITIVE_WRAPPER_TYPE); } -TNode<BoolT> CodeStubAssembler::IsJSValue(SloppyTNode<HeapObject> object) { - return IsJSValueMap(LoadMap(object)); +TNode<BoolT> CodeStubAssembler::IsJSPrimitiveWrapper( + SloppyTNode<HeapObject> object) { + return IsJSPrimitiveWrapperMap(LoadMap(object)); } -TNode<BoolT> CodeStubAssembler::IsJSValueMap(SloppyTNode<Map> map) { - return IsJSValueInstanceType(LoadMapInstanceType(map)); +TNode<BoolT> CodeStubAssembler::IsJSPrimitiveWrapperMap(SloppyTNode<Map> map) { + return IsJSPrimitiveWrapperInstanceType(LoadMapInstanceType(map)); } TNode<BoolT> CodeStubAssembler::IsJSArrayInstanceType( @@ -6420,7 +6454,7 @@ TNode<BoolT> CodeStubAssembler::IsFixedArrayWithKind( if (IsDoubleElementsKind(kind)) { return IsFixedDoubleArray(object); } else { - DCHECK(IsSmiOrObjectElementsKind(kind)); + DCHECK(IsSmiOrObjectElementsKind(kind) || IsSealedElementsKind(kind)); return IsFixedArraySubclass(object); } } @@ -6562,6 +6596,11 @@ TNode<BoolT> CodeStubAssembler::IsPrivateSymbol( [=] { return Int32FalseConstant(); }); } +TNode<BoolT> CodeStubAssembler::IsPrivateName(SloppyTNode<Symbol> symbol) { + TNode<Uint32T> flags = LoadObjectField<Uint32T>(symbol, Symbol::kFlagsOffset); + return IsSetWord32<Symbol::IsPrivateNameBit>(flags); +} + TNode<BoolT> CodeStubAssembler::IsNativeContext( SloppyTNode<HeapObject> object) { return WordEqual(LoadMap(object), LoadRoot(RootIndex::kNativeContextMap)); @@ -6769,7 +6808,7 @@ TNode<BoolT> CodeStubAssembler::IsHeapNumberUint32(TNode<HeapNumber> number) { IsHeapNumberPositive(number), [=] { TNode<Float64T> value = LoadHeapNumberValue(number); - TNode<Uint32T> int_value = Unsigned(TruncateFloat64ToWord32(value)); + TNode<Uint32T> int_value = TruncateFloat64ToWord32(value); return Float64Equal(value, ChangeUint32ToFloat64(int_value)); }, [=] { return Int32FalseConstant(); }); @@ -7423,8 +7462,8 @@ TNode<String> CodeStubAssembler::StringAdd(Node* context, TNode<String> left, return result.value(); } -TNode<String> CodeStubAssembler::StringFromSingleCodePoint( - TNode<Int32T> codepoint, UnicodeEncoding encoding) { +TNode<String> CodeStubAssembler::StringFromSingleUTF16EncodedCodePoint( + TNode<Int32T> codepoint) { VARIABLE(var_result, MachineRepresentation::kTagged, EmptyStringConstant()); Label if_isword16(this), if_isword32(this), return_result(this); @@ -7440,27 +7479,6 @@ TNode<String> CodeStubAssembler::StringFromSingleCodePoint( BIND(&if_isword32); { - switch (encoding) { - case UnicodeEncoding::UTF16: - break; - case UnicodeEncoding::UTF32: { - // Convert UTF32 to UTF16 code units, and store as a 32 bit word. - Node* lead_offset = Int32Constant(0xD800 - (0x10000 >> 10)); - - // lead = (codepoint >> 10) + LEAD_OFFSET - Node* lead = - Int32Add(Word32Shr(codepoint, Int32Constant(10)), lead_offset); - - // trail = (codepoint & 0x3FF) + 0xDC00; - Node* trail = Int32Add(Word32And(codepoint, Int32Constant(0x3FF)), - Int32Constant(0xDC00)); - - // codpoint = (trail << 16) | lead; - codepoint = Signed(Word32Or(Word32Shl(trail, Int32Constant(16)), lead)); - break; - } - } - Node* value = AllocateSeqTwoByteString(2); StoreNoWriteBarrier( MachineRepresentation::kWord32, value, @@ -7513,7 +7531,7 @@ TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { // contains two elements (number and string) for each cache entry. // TODO(ishell): cleanup mask handling. Node* mask = - BitcastTaggedToWord(LoadFixedArrayBaseLength(number_string_cache)); + BitcastTaggedSignedToWord(LoadFixedArrayBaseLength(number_string_cache)); TNode<IntPtrT> one = IntPtrConstant(1); mask = IntPtrSub(mask, one); @@ -7560,8 +7578,8 @@ TNode<String> CodeStubAssembler::NumberToString(TNode<Number> input) { BIND(&if_smi); { // Load the smi key, make sure it matches the smi we're looking for. - Node* smi_index = BitcastWordToTagged( - WordAnd(WordShl(BitcastTaggedToWord(smi_input.value()), one), mask)); + Node* smi_index = BitcastWordToTagged(WordAnd( + WordShl(BitcastTaggedSignedToWord(smi_input.value()), one), mask)); Node* smi_key = UnsafeLoadFixedArrayElement(CAST(number_string_cache), smi_index, 0, SMI_PARAMETERS); GotoIf(WordNotEqual(smi_key, smi_input.value()), &runtime); @@ -8333,40 +8351,41 @@ TNode<IntPtrT> CodeStubAssembler::EntryToIndex(TNode<IntPtrT> entry, field_index)); } -TNode<MaybeObject> CodeStubAssembler::LoadDescriptorArrayElement( - TNode<DescriptorArray> object, Node* index, int additional_offset) { - return LoadArrayElement(object, DescriptorArray::kHeaderSize, index, - additional_offset); +template <typename T> +TNode<T> CodeStubAssembler::LoadDescriptorArrayElement( + TNode<DescriptorArray> object, TNode<IntPtrT> index, + int additional_offset) { + return LoadArrayElement<DescriptorArray, T>( + object, DescriptorArray::kHeaderSize, index, additional_offset); } TNode<Name> CodeStubAssembler::LoadKeyByKeyIndex( TNode<DescriptorArray> container, TNode<IntPtrT> key_index) { - return CAST(LoadDescriptorArrayElement(container, key_index, 0)); + return CAST(LoadDescriptorArrayElement<HeapObject>(container, key_index, 0)); } TNode<Uint32T> CodeStubAssembler::LoadDetailsByKeyIndex( TNode<DescriptorArray> container, TNode<IntPtrT> key_index) { - const int kKeyToDetails = - DescriptorArray::ToDetailsIndex(0) - DescriptorArray::ToKeyIndex(0); - return Unsigned( - LoadAndUntagToWord32ArrayElement(container, DescriptorArray::kHeaderSize, - key_index, kKeyToDetails * kTaggedSize)); + const int kKeyToDetailsOffset = + DescriptorArray::kEntryDetailsOffset - DescriptorArray::kEntryKeyOffset; + return Unsigned(LoadAndUntagToWord32ArrayElement( + container, DescriptorArray::kHeaderSize, key_index, kKeyToDetailsOffset)); } TNode<Object> CodeStubAssembler::LoadValueByKeyIndex( TNode<DescriptorArray> container, TNode<IntPtrT> key_index) { - const int kKeyToValue = - DescriptorArray::ToValueIndex(0) - DescriptorArray::ToKeyIndex(0); - return CAST(LoadDescriptorArrayElement(container, key_index, - kKeyToValue * kTaggedSize)); + const int kKeyToValueOffset = + DescriptorArray::kEntryValueOffset - DescriptorArray::kEntryKeyOffset; + return LoadDescriptorArrayElement<Object>(container, key_index, + kKeyToValueOffset); } TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByKeyIndex( TNode<DescriptorArray> container, TNode<IntPtrT> key_index) { - const int kKeyToValue = - DescriptorArray::ToValueIndex(0) - DescriptorArray::ToKeyIndex(0); - return LoadDescriptorArrayElement(container, key_index, - kKeyToValue * kTaggedSize); + const int kKeyToValueOffset = + DescriptorArray::kEntryValueOffset - DescriptorArray::kEntryKeyOffset; + return LoadDescriptorArrayElement<MaybeObject>(container, key_index, + kKeyToValueOffset); } TNode<IntPtrT> CodeStubAssembler::DescriptorEntryToIndex( @@ -8377,14 +8396,14 @@ TNode<IntPtrT> CodeStubAssembler::DescriptorEntryToIndex( TNode<Name> CodeStubAssembler::LoadKeyByDescriptorEntry( TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) { - return CAST(LoadDescriptorArrayElement( + return CAST(LoadDescriptorArrayElement<HeapObject>( container, DescriptorEntryToIndex(descriptor_entry), DescriptorArray::ToKeyIndex(0) * kTaggedSize)); } TNode<Name> CodeStubAssembler::LoadKeyByDescriptorEntry( TNode<DescriptorArray> container, int descriptor_entry) { - return CAST(LoadDescriptorArrayElement( + return CAST(LoadDescriptorArrayElement<HeapObject>( container, IntPtrConstant(0), DescriptorArray::ToKeyIndex(descriptor_entry) * kTaggedSize)); } @@ -8406,14 +8425,14 @@ TNode<Uint32T> CodeStubAssembler::LoadDetailsByDescriptorEntry( TNode<Object> CodeStubAssembler::LoadValueByDescriptorEntry( TNode<DescriptorArray> container, int descriptor_entry) { - return CAST(LoadDescriptorArrayElement( + return LoadDescriptorArrayElement<Object>( container, IntPtrConstant(0), - DescriptorArray::ToValueIndex(descriptor_entry) * kTaggedSize)); + DescriptorArray::ToValueIndex(descriptor_entry) * kTaggedSize); } TNode<MaybeObject> CodeStubAssembler::LoadFieldTypeByDescriptorEntry( TNode<DescriptorArray> container, TNode<IntPtrT> descriptor_entry) { - return LoadDescriptorArrayElement( + return LoadDescriptorArrayElement<MaybeObject>( container, DescriptorEntryToIndex(descriptor_entry), DescriptorArray::ToValueIndex(0) * kTaggedSize); } @@ -9503,15 +9522,15 @@ TNode<Object> CodeStubAssembler::CallGetterIfAccessor( Node* accessor_info = value; CSA_ASSERT(this, IsAccessorInfo(value)); CSA_ASSERT(this, TaggedIsNotSmi(receiver)); - Label if_array(this), if_function(this), if_value(this); + Label if_array(this), if_function(this), if_wrapper(this); // Dispatch based on {receiver} instance type. Node* receiver_map = LoadMap(receiver); Node* receiver_instance_type = LoadMapInstanceType(receiver_map); GotoIf(IsJSArrayInstanceType(receiver_instance_type), &if_array); GotoIf(IsJSFunctionInstanceType(receiver_instance_type), &if_function); - Branch(IsJSValueInstanceType(receiver_instance_type), &if_value, - if_bailout); + Branch(IsJSPrimitiveWrapperInstanceType(receiver_instance_type), + &if_wrapper, if_bailout); // JSArray AccessorInfo case. BIND(&if_array); @@ -9538,14 +9557,15 @@ TNode<Object> CodeStubAssembler::CallGetterIfAccessor( Goto(&done); } - // JSValue AccessorInfo case. - BIND(&if_value); + // JSPrimitiveWrapper AccessorInfo case. + BIND(&if_wrapper); { - // We only deal with the "length" accessor on JSValue string wrappers. + // We only deal with the "length" accessor on JSPrimitiveWrapper string + // wrappers. GotoIfNot(IsLengthString( LoadObjectField(accessor_info, AccessorInfo::kNameOffset)), if_bailout); - Node* receiver_value = LoadJSValueValue(receiver); + Node* receiver_value = LoadJSPrimitiveWrapperValue(receiver); GotoIfNot(TaggedIsNotSmi(receiver_value), if_bailout); GotoIfNot(IsString(receiver_value), if_bailout); var_value.Bind(LoadStringLengthAsSmi(receiver_value)); @@ -9646,8 +9666,9 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, // clang-format off int32_t values[] = { // Handled by {if_isobjectorsmi}. - PACKED_SMI_ELEMENTS, HOLEY_SMI_ELEMENTS, PACKED_ELEMENTS, - HOLEY_ELEMENTS, + PACKED_SMI_ELEMENTS, HOLEY_SMI_ELEMENTS, PACKED_ELEMENTS, HOLEY_ELEMENTS, + PACKED_SEALED_ELEMENTS, HOLEY_SEALED_ELEMENTS, PACKED_FROZEN_ELEMENTS, + HOLEY_FROZEN_ELEMENTS, // Handled by {if_isdouble}. PACKED_DOUBLE_ELEMENTS, HOLEY_DOUBLE_ELEMENTS, // Handled by {if_isdictionary}. @@ -9673,7 +9694,8 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, }; Label* labels[] = { &if_isobjectorsmi, &if_isobjectorsmi, &if_isobjectorsmi, - &if_isobjectorsmi, + &if_isobjectorsmi, &if_isobjectorsmi, &if_isobjectorsmi, + &if_isobjectorsmi, &if_isobjectorsmi, &if_isdouble, &if_isdouble, &if_isdictionary, &if_isfaststringwrapper, @@ -9731,8 +9753,8 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, } BIND(&if_isfaststringwrapper); { - CSA_ASSERT(this, HasInstanceType(object, JS_VALUE_TYPE)); - Node* string = LoadJSValueValue(object); + CSA_ASSERT(this, HasInstanceType(object, JS_PRIMITIVE_WRAPPER_TYPE)); + Node* string = LoadJSPrimitiveWrapperValue(object); CSA_ASSERT(this, IsString(string)); Node* length = LoadStringLengthAsWord(string); GotoIf(UintPtrLessThan(intptr_index, length), if_found); @@ -9740,8 +9762,8 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, } BIND(&if_isslowstringwrapper); { - CSA_ASSERT(this, HasInstanceType(object, JS_VALUE_TYPE)); - Node* string = LoadJSValueValue(object); + CSA_ASSERT(this, HasInstanceType(object, JS_PRIMITIVE_WRAPPER_TYPE)); + Node* string = LoadJSPrimitiveWrapperValue(object); CSA_ASSERT(this, IsString(string)); Node* length = LoadStringLengthAsWord(string); GotoIf(UintPtrLessThan(intptr_index, length), if_found); @@ -9749,7 +9771,7 @@ void CodeStubAssembler::TryLookupElement(Node* object, Node* map, } BIND(&if_typedarray); { - Node* buffer = LoadObjectField(object, JSArrayBufferView::kBufferOffset); + TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(CAST(object)); GotoIf(IsDetachedBuffer(buffer), if_absent); TNode<UintPtrT> length = LoadJSTypedArrayLength(CAST(object)); @@ -9794,15 +9816,15 @@ void CodeStubAssembler::BranchIfMaybeSpecialIndex(TNode<String> name_string, } void CodeStubAssembler::TryPrototypeChainLookup( - Node* receiver, Node* key, const LookupInHolder& lookup_property_in_holder, + Node* receiver, Node* object, Node* key, + const LookupInHolder& lookup_property_in_holder, const LookupInHolder& lookup_element_in_holder, Label* if_end, Label* if_bailout, Label* if_proxy) { // Ensure receiver is JSReceiver, otherwise bailout. - Label if_objectisnotsmi(this); - Branch(TaggedIsSmi(receiver), if_bailout, &if_objectisnotsmi); - BIND(&if_objectisnotsmi); + GotoIf(TaggedIsSmi(receiver), if_bailout); + CSA_ASSERT(this, TaggedIsNotSmi(object)); - Node* map = LoadMap(receiver); + Node* map = LoadMap(object); Node* instance_type = LoadMapInstanceType(map); { Label if_objectisreceiver(this); @@ -9812,9 +9834,7 @@ void CodeStubAssembler::TryPrototypeChainLookup( if_bailout); BIND(&if_objectisreceiver); - if (if_proxy) { - GotoIf(InstanceTypeEqual(instance_type, JS_PROXY_TYPE), if_proxy); - } + GotoIf(InstanceTypeEqual(instance_type, JS_PROXY_TYPE), if_proxy); } VARIABLE(var_index, MachineType::PointerRepresentation()); @@ -9826,7 +9846,7 @@ void CodeStubAssembler::TryPrototypeChainLookup( BIND(&if_iskeyunique); { - VARIABLE(var_holder, MachineRepresentation::kTagged, receiver); + VARIABLE(var_holder, MachineRepresentation::kTagged, object); VARIABLE(var_holder_map, MachineRepresentation::kTagged, map); VARIABLE(var_holder_instance_type, MachineRepresentation::kWord32, instance_type); @@ -9872,7 +9892,7 @@ void CodeStubAssembler::TryPrototypeChainLookup( } BIND(&if_keyisindex); { - VARIABLE(var_holder, MachineRepresentation::kTagged, receiver); + VARIABLE(var_holder, MachineRepresentation::kTagged, object); VARIABLE(var_holder_map, MachineRepresentation::kTagged, map); VARIABLE(var_holder_instance_type, MachineRepresentation::kWord32, instance_type); @@ -10049,7 +10069,7 @@ TNode<IntPtrT> CodeStubAssembler::ElementOffsetFromIndex(Node* index_node, Smi smi_index; constant_index = ToSmiConstant(index_node, &smi_index); if (constant_index) index = smi_index.value(); - index_node = BitcastTaggedToWord(index_node); + index_node = BitcastTaggedSignedToWord(index_node); } else { DCHECK(mode == INTPTR_PARAMETERS); constant_index = ToIntPtrConstant(index_node, index); @@ -10594,7 +10614,8 @@ void CodeStubAssembler::BigIntToRawBytes(TNode<BigInt> bigint, void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, ElementsKind elements_kind, KeyedAccessStoreMode store_mode, - Label* bailout, Node* context) { + Label* bailout, Node* context, + Variable* maybe_converted_value) { CSA_ASSERT(this, Word32BinaryNot(IsJSProxy(object))); Node* elements = LoadElements(object); @@ -10610,12 +10631,12 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, TNode<IntPtrT> intptr_key = TryToIntptr(key, bailout); if (IsTypedArrayElementsKind(elements_kind)) { - Label done(this); + Label done(this), update_value_and_bailout(this, Label::kDeferred); // IntegerIndexedElementSet converts value to a Number/BigInt prior to the // bounds check. - value = PrepareValueForWriteToTypedArray(CAST(value), elements_kind, - CAST(context)); + Node* converted_value = PrepareValueForWriteToTypedArray( + CAST(value), elements_kind, CAST(context)); // There must be no allocations between the buffer load and // and the actual store to backing store, because GC may decide that @@ -10623,8 +10644,12 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, // TODO(ishell): introduce DisallowHeapAllocationCode scope here. // Check if buffer has been detached. - Node* buffer = LoadObjectField(object, JSArrayBufferView::kBufferOffset); - GotoIf(IsDetachedBuffer(buffer), bailout); + TNode<JSArrayBuffer> buffer = LoadJSArrayBufferViewBuffer(CAST(object)); + if (maybe_converted_value) { + GotoIf(IsDetachedBuffer(buffer), &update_value_and_bailout); + } else { + GotoIf(IsDetachedBuffer(buffer), bailout); + } // Bounds check. TNode<UintPtrT> length = LoadJSTypedArrayLength(CAST(object)); @@ -10633,27 +10658,88 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, // Skip the store if we write beyond the length or // to a property with a negative integer index. GotoIfNot(UintPtrLessThan(intptr_key, length), &done); - } else if (store_mode == STANDARD_STORE) { - GotoIfNot(UintPtrLessThan(intptr_key, length), bailout); } else { - // This case is produced due to the dispatched call in - // ElementsTransitionAndStore and StoreFastElement. - // TODO(jgruber): Avoid generating unsupported combinations to save code - // size. - DebugBreak(); + DCHECK_EQ(store_mode, STANDARD_STORE); + GotoIfNot(UintPtrLessThan(intptr_key, length), &update_value_and_bailout); } TNode<RawPtrT> backing_store = LoadJSTypedArrayBackingStore(CAST(object)); - StoreElement(backing_store, elements_kind, intptr_key, value, + StoreElement(backing_store, elements_kind, intptr_key, converted_value, parameter_mode); Goto(&done); + BIND(&update_value_and_bailout); + // We already prepared the incoming value for storing into a typed array. + // This might involve calling ToNumber in some cases. We shouldn't call + // ToNumber again in the runtime so pass the converted value to the runtime. + // The prepared value is an untagged value. Convert it to a tagged value + // to pass it to runtime. It is not possible to do the detached buffer check + // before we prepare the value, since ToNumber can detach the ArrayBuffer. + // The spec specifies the order of these operations. + if (maybe_converted_value != nullptr) { + switch (elements_kind) { + case UINT8_ELEMENTS: + case INT8_ELEMENTS: + case UINT16_ELEMENTS: + case INT16_ELEMENTS: + case UINT8_CLAMPED_ELEMENTS: + maybe_converted_value->Bind(SmiFromInt32(converted_value)); + break; + case UINT32_ELEMENTS: + maybe_converted_value->Bind(ChangeUint32ToTagged(converted_value)); + break; + case INT32_ELEMENTS: + maybe_converted_value->Bind(ChangeInt32ToTagged(converted_value)); + break; + case FLOAT32_ELEMENTS: { + Label dont_allocate_heap_number(this), end(this); + GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number); + GotoIf(IsHeapNumber(value), &dont_allocate_heap_number); + { + maybe_converted_value->Bind(AllocateHeapNumberWithValue( + ChangeFloat32ToFloat64(converted_value))); + Goto(&end); + } + BIND(&dont_allocate_heap_number); + { + maybe_converted_value->Bind(value); + Goto(&end); + } + BIND(&end); + break; + } + case FLOAT64_ELEMENTS: { + Label dont_allocate_heap_number(this), end(this); + GotoIf(TaggedIsSmi(value), &dont_allocate_heap_number); + GotoIf(IsHeapNumber(value), &dont_allocate_heap_number); + { + maybe_converted_value->Bind( + AllocateHeapNumberWithValue(converted_value)); + Goto(&end); + } + BIND(&dont_allocate_heap_number); + { + maybe_converted_value->Bind(value); + Goto(&end); + } + BIND(&end); + break; + } + case BIGINT64_ELEMENTS: + case BIGUINT64_ELEMENTS: + maybe_converted_value->Bind(converted_value); + break; + default: + UNREACHABLE(); + } + } + Goto(bailout); + BIND(&done); return; } - DCHECK( - IsFastElementsKind(elements_kind) || - IsInRange(elements_kind, PACKED_SEALED_ELEMENTS, HOLEY_SEALED_ELEMENTS)); + DCHECK(IsFastElementsKind(elements_kind) || + IsSealedElementsKind(elements_kind)); Node* length = SelectImpl( IsJSArray(object), [=]() { return LoadJSArrayLength(object); }, @@ -10670,18 +10756,24 @@ void CodeStubAssembler::EmitElementStore(Node* object, Node* key, Node* value, value = TryTaggedToFloat64(value, bailout); } - if (IsGrowStoreMode(store_mode) && - !(IsInRange(elements_kind, PACKED_SEALED_ELEMENTS, - HOLEY_SEALED_ELEMENTS))) { + if (IsGrowStoreMode(store_mode) && !IsSealedElementsKind(elements_kind)) { elements = CheckForCapacityGrow(object, elements, elements_kind, length, intptr_key, parameter_mode, bailout); } else { GotoIfNot(UintPtrLessThan(intptr_key, length), bailout); } + // Cannot store to a hole in holey sealed elements so bailout. + if (elements_kind == HOLEY_SEALED_ELEMENTS) { + TNode<Object> target_value = + LoadFixedArrayElement(CAST(elements), intptr_key); + GotoIf(IsTheHole(target_value), bailout); + } + // If we didn't grow {elements}, it might still be COW, in which case we // copy it now. - if (!IsSmiOrObjectElementsKind(elements_kind)) { + if (!(IsSmiOrObjectElementsKind(elements_kind) || + IsSealedElementsKind(elements_kind))) { CSA_ASSERT(this, Word32BinaryNot(IsFixedCOWArrayMap(LoadMap(elements)))); } else if (IsCOWHandlingStoreMode(store_mode)) { elements = CopyElementsOnWrite(object, elements, elements_kind, length, @@ -10925,7 +11017,8 @@ TNode<AllocationSite> CodeStubAssembler::CreateAllocationSiteInFeedbackVector( // Link the object to the allocation site list TNode<ExternalReference> site_list = ExternalConstant( ExternalReference::allocation_sites_list_address(isolate())); - TNode<Object> next_site = CAST(LoadBufferObject(site_list, 0)); + TNode<Object> next_site = + LoadBufferObject(ReinterpretCast<RawPtrT>(site_list), 0); // TODO(mvstanton): This is a store to a weak pointer, which we may want to // mark as such in order to skip the write barrier, once we have a unified @@ -12155,8 +12248,9 @@ Node* CodeStubAssembler::Equal(Node* left, Node* right, Node* context, return result.value(); } -Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, - Variable* var_type_feedback) { +TNode<Oddball> CodeStubAssembler::StrictEqual(SloppyTNode<Object> lhs, + SloppyTNode<Object> rhs, + Variable* var_type_feedback) { // Pseudo-code for the algorithm below: // // if (lhs == rhs) { @@ -12208,7 +12302,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, Label if_equal(this), if_notequal(this), if_not_equivalent_types(this), end(this); - VARIABLE(result, MachineRepresentation::kTagged); + TVARIABLE(Oddball, result); OverwriteFeedback(var_type_feedback, CompareOperationFeedback::kNone); @@ -12235,7 +12329,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_lhsisnotsmi); { // Load the map of {lhs}. - Node* lhs_map = LoadMap(lhs); + TNode<Map> lhs_map = LoadMap(CAST(lhs)); // Check if {lhs} is a HeapNumber. Label if_lhsisnumber(this), if_lhsisnotnumber(this); @@ -12250,8 +12344,8 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_rhsissmi); { // Convert {lhs} and {rhs} to floating point values. - Node* lhs_value = LoadHeapNumberValue(lhs); - Node* rhs_value = SmiToFloat64(rhs); + Node* lhs_value = LoadHeapNumberValue(CAST(lhs)); + Node* rhs_value = SmiToFloat64(CAST(rhs)); CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber); @@ -12261,8 +12355,9 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_rhsisnotsmi); { + TNode<HeapObject> rhs_ho = CAST(rhs); // Load the map of {rhs}. - Node* rhs_map = LoadMap(rhs); + TNode<Map> rhs_map = LoadMap(rhs_ho); // Check if {rhs} is also a HeapNumber. Label if_rhsisnumber(this), if_rhsisnotnumber(this); @@ -12271,8 +12366,8 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_rhsisnumber); { // Convert {lhs} and {rhs} to floating point values. - Node* lhs_value = LoadHeapNumberValue(lhs); - Node* rhs_value = LoadHeapNumberValue(rhs); + Node* lhs_value = LoadHeapNumberValue(CAST(lhs)); + Node* rhs_value = LoadHeapNumberValue(CAST(rhs)); CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber); @@ -12308,7 +12403,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_lhsisstring); { // Load the instance type of {rhs}. - Node* rhs_instance_type = LoadInstanceType(rhs); + Node* rhs_instance_type = LoadInstanceType(CAST(rhs)); // Check if {rhs} is also a String. Label if_rhsisstring(this, Label::kDeferred), @@ -12325,8 +12420,8 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, CollectFeedbackForString(rhs_instance_type); var_type_feedback->Bind(SmiOr(lhs_feedback, rhs_feedback)); } - result.Bind(CallBuiltin(Builtins::kStringEqual, - NoContextConstant(), lhs, rhs)); + result = CAST(CallBuiltin(Builtins::kStringEqual, + NoContextConstant(), lhs, rhs)); Goto(&end); } @@ -12344,7 +12439,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_lhsisbigint); { // Load the instance type of {rhs}. - Node* rhs_instance_type = LoadInstanceType(rhs); + TNode<Uint16T> rhs_instance_type = LoadInstanceType(CAST(rhs)); // Check if {rhs} is also a BigInt. Label if_rhsisbigint(this, Label::kDeferred), @@ -12356,8 +12451,8 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, { CombineFeedback(var_type_feedback, CompareOperationFeedback::kBigInt); - result.Bind(CallRuntime(Runtime::kBigIntEqualToBigInt, - NoContextConstant(), lhs, rhs)); + result = CAST(CallRuntime(Runtime::kBigIntEqualToBigInt, + NoContextConstant(), lhs, rhs)); Goto(&end); } @@ -12368,8 +12463,8 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_lhsisnotbigint); if (var_type_feedback != nullptr) { // Load the instance type of {rhs}. - Node* rhs_map = LoadMap(rhs); - Node* rhs_instance_type = LoadMapInstanceType(rhs_map); + TNode<Map> rhs_map = LoadMap(CAST(rhs)); + TNode<Uint16T> rhs_instance_type = LoadMapInstanceType(rhs_map); Label if_lhsissymbol(this), if_lhsisreceiver(this), if_lhsisoddball(this); @@ -12442,7 +12537,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_rhsisnotsmi); { // Load the map of the {rhs}. - Node* rhs_map = LoadMap(rhs); + TNode<Map> rhs_map = LoadMap(CAST(rhs)); // The {rhs} could be a HeapNumber with the same value as {lhs}. Label if_rhsisnumber(this), if_rhsisnotnumber(this); @@ -12451,8 +12546,8 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_rhsisnumber); { // Convert {lhs} and {rhs} to floating point values. - Node* lhs_value = SmiToFloat64(lhs); - Node* rhs_value = LoadHeapNumberValue(rhs); + TNode<Float64T> lhs_value = SmiToFloat64(CAST(lhs)); + TNode<Float64T> rhs_value = LoadHeapNumberValue(CAST(rhs)); CombineFeedback(var_type_feedback, CompareOperationFeedback::kNumber); @@ -12468,7 +12563,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_equal); { - result.Bind(TrueConstant()); + result = TrueConstant(); Goto(&end); } @@ -12480,7 +12575,7 @@ Node* CodeStubAssembler::StrictEqual(Node* lhs, Node* rhs, BIND(&if_notequal); { - result.Bind(FalseConstant()); + result = FalseConstant(); Goto(&end); } @@ -12636,7 +12731,7 @@ TNode<Oddball> CodeStubAssembler::HasProperty(SloppyTNode<Context> context, &return_true, &return_false, next_holder, if_bailout); }; - TryPrototypeChainLookup(object, key, lookup_property_in_holder, + TryPrototypeChainLookup(object, object, key, lookup_property_in_holder, lookup_element_in_holder, &return_false, &call_runtime, &if_proxy); @@ -13114,8 +13209,9 @@ TNode<JSArrayIterator> CodeStubAssembler::CreateArrayIterator( return CAST(iterator); } -Node* CodeStubAssembler::AllocateJSIteratorResult(Node* context, Node* value, - Node* done) { +TNode<JSObject> CodeStubAssembler::AllocateJSIteratorResult( + SloppyTNode<Context> context, SloppyTNode<Object> value, + SloppyTNode<Oddball> done) { CSA_ASSERT(this, IsBoolean(done)); Node* native_context = LoadNativeContext(context); Node* map = @@ -13128,7 +13224,7 @@ Node* CodeStubAssembler::AllocateJSIteratorResult(Node* context, Node* value, RootIndex::kEmptyFixedArray); StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kValueOffset, value); StoreObjectFieldNoWriteBarrier(result, JSIteratorResult::kDoneOffset, done); - return result; + return CAST(result); } Node* CodeStubAssembler::AllocateJSIteratorResultForEntry(Node* context, @@ -13174,9 +13270,8 @@ TNode<JSReceiver> CodeStubAssembler::ArraySpeciesCreate(TNode<Context> context, return Construct(context, constructor, len); } -Node* CodeStubAssembler::IsDetachedBuffer(Node* buffer) { - CSA_ASSERT(this, HasInstanceType(buffer, JS_ARRAY_BUFFER_TYPE)); - TNode<Uint32T> buffer_bit_field = LoadJSArrayBufferBitField(CAST(buffer)); +TNode<BoolT> CodeStubAssembler::IsDetachedBuffer(TNode<JSArrayBuffer> buffer) { + TNode<Uint32T> buffer_bit_field = LoadJSArrayBufferBitField(buffer); return IsSetWord32<JSArrayBuffer::WasDetachedBit>(buffer_bit_field); } @@ -13367,7 +13462,8 @@ void CodeStubArguments::PopAndReturn(Node* value) { value); } -Node* CodeStubAssembler::IsFastElementsKind(Node* elements_kind) { +TNode<BoolT> CodeStubAssembler::IsFastElementsKind( + TNode<Int32T> elements_kind) { STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND); return Uint32LessThanOrEqual(elements_kind, Int32Constant(LAST_FAST_ELEMENTS_KIND)); @@ -13382,7 +13478,8 @@ TNode<BoolT> CodeStubAssembler::IsDoubleElementsKind( Int32Constant(PACKED_DOUBLE_ELEMENTS / 2)); } -Node* CodeStubAssembler::IsFastSmiOrTaggedElementsKind(Node* elements_kind) { +TNode<BoolT> CodeStubAssembler::IsFastSmiOrTaggedElementsKind( + TNode<Int32T> elements_kind) { STATIC_ASSERT(FIRST_ELEMENTS_KIND == FIRST_FAST_ELEMENTS_KIND); STATIC_ASSERT(PACKED_DOUBLE_ELEMENTS > TERMINAL_FAST_ELEMENTS_KIND); STATIC_ASSERT(HOLEY_DOUBLE_ELEMENTS > TERMINAL_FAST_ELEMENTS_KIND); @@ -13390,12 +13487,14 @@ Node* CodeStubAssembler::IsFastSmiOrTaggedElementsKind(Node* elements_kind) { Int32Constant(TERMINAL_FAST_ELEMENTS_KIND)); } -Node* CodeStubAssembler::IsFastSmiElementsKind(Node* elements_kind) { +TNode<BoolT> CodeStubAssembler::IsFastSmiElementsKind( + SloppyTNode<Int32T> elements_kind) { return Uint32LessThanOrEqual(elements_kind, Int32Constant(HOLEY_SMI_ELEMENTS)); } -Node* CodeStubAssembler::IsHoleyFastElementsKind(Node* elements_kind) { +TNode<BoolT> CodeStubAssembler::IsHoleyFastElementsKind( + TNode<Int32T> elements_kind) { CSA_ASSERT(this, IsFastElementsKind(elements_kind)); STATIC_ASSERT(HOLEY_SMI_ELEMENTS == (PACKED_SMI_ELEMENTS | 1)); @@ -13404,7 +13503,8 @@ Node* CodeStubAssembler::IsHoleyFastElementsKind(Node* elements_kind) { return IsSetWord32(elements_kind, 1); } -Node* CodeStubAssembler::IsHoleyFastElementsKindForRead(Node* elements_kind) { +TNode<BoolT> CodeStubAssembler::IsHoleyFastElementsKindForRead( + TNode<Int32T> elements_kind) { CSA_ASSERT(this, Uint32LessThanOrEqual(elements_kind, Int32Constant(LAST_FROZEN_ELEMENTS_KIND))); @@ -13417,8 +13517,8 @@ Node* CodeStubAssembler::IsHoleyFastElementsKindForRead(Node* elements_kind) { return IsSetWord32(elements_kind, 1); } -Node* CodeStubAssembler::IsElementsKindGreaterThan( - Node* target_kind, ElementsKind reference_kind) { +TNode<BoolT> CodeStubAssembler::IsElementsKindGreaterThan( + TNode<Int32T> target_kind, ElementsKind reference_kind) { return Int32GreaterThan(target_kind, Int32Constant(reference_kind)); } @@ -13442,14 +13542,6 @@ Node* CodeStubAssembler::IsDebugActive() { return Word32NotEqual(is_debug_active, Int32Constant(0)); } -TNode<BoolT> CodeStubAssembler::IsRuntimeCallStatsEnabled() { - STATIC_ASSERT(sizeof(TracingFlags::runtime_stats) == kInt32Size); - TNode<Word32T> flag_value = UncheckedCast<Word32T>(Load( - MachineType::Int32(), - ExternalConstant(ExternalReference::address_of_runtime_stats_flag()))); - return Word32NotEqual(flag_value, Int32Constant(0)); -} - Node* CodeStubAssembler::IsPromiseHookEnabled() { Node* const promise_hook = Load( MachineType::Pointer(), @@ -13494,8 +13586,9 @@ TNode<Code> CodeStubAssembler::LoadBuiltin(TNode<Smi> builtin_id) { int const kSmiShiftBits = kSmiShiftSize + kSmiTagSize; int index_shift = kSystemPointerSizeLog2 - kSmiShiftBits; TNode<WordT> table_index = - index_shift >= 0 ? WordShl(BitcastTaggedToWord(builtin_id), index_shift) - : WordSar(BitcastTaggedToWord(builtin_id), -index_shift); + index_shift >= 0 + ? WordShl(BitcastTaggedSignedToWord(builtin_id), index_shift) + : WordSar(BitcastTaggedSignedToWord(builtin_id), -index_shift); return CAST( Load(MachineType::TaggedPointer(), @@ -13637,18 +13730,6 @@ Node* CodeStubAssembler::AllocateFunctionWithMapAndContext(Node* map, return fun; } -Node* CodeStubAssembler::MarkerIsFrameType(Node* marker_or_function, - StackFrame::Type frame_type) { - return WordEqual(marker_or_function, - IntPtrConstant(StackFrame::TypeToMarker(frame_type))); -} - -Node* CodeStubAssembler::MarkerIsNotFrameType(Node* marker_or_function, - StackFrame::Type frame_type) { - return WordNotEqual(marker_or_function, - IntPtrConstant(StackFrame::TypeToMarker(frame_type))); -} - void CodeStubAssembler::CheckPrototypeEnumCache(Node* receiver, Node* receiver_map, Label* if_fast, @@ -13923,7 +14004,7 @@ void CodeStubAssembler::GotoIfInitialPrototypePropertiesModified( if (i == 0) { combined_details = details; } else { - combined_details = Unsigned(Word32And(combined_details, details)); + combined_details = Word32And(combined_details, details); } } |