summaryrefslogtreecommitdiff
path: root/deps/v8/src/builtins/arm64/builtins-arm64.cc
diff options
context:
space:
mode:
Diffstat (limited to 'deps/v8/src/builtins/arm64/builtins-arm64.cc')
-rw-r--r--deps/v8/src/builtins/arm64/builtins-arm64.cc246
1 files changed, 117 insertions, 129 deletions
diff --git a/deps/v8/src/builtins/arm64/builtins-arm64.cc b/deps/v8/src/builtins/arm64/builtins-arm64.cc
index 4e159a69b7..9edd074e3d 100644
--- a/deps/v8/src/builtins/arm64/builtins-arm64.cc
+++ b/deps/v8/src/builtins/arm64/builtins-arm64.cc
@@ -1001,108 +1001,78 @@ static void TailCallRuntimeIfMarkerEquals(MacroAssembler* masm,
OptimizationMarker marker,
Runtime::FunctionId function_id) {
Label no_match;
- __ CompareAndBranch(smi_entry, Operand(Smi::FromEnum(marker)), ne, &no_match);
+ __ CompareTaggedAndBranch(smi_entry, Operand(Smi::FromEnum(marker)), ne,
+ &no_match);
GenerateTailCallToReturnedCode(masm, function_id);
__ bind(&no_match);
}
-static void MaybeTailCallOptimizedCodeSlot(MacroAssembler* masm,
- Register feedback_vector,
- Register scratch1,
- Register scratch2) {
+static void TailCallOptimizedCodeSlot(MacroAssembler* masm,
+ Register optimized_code_entry,
+ Register scratch) {
// ----------- S t a t e -------------
// -- x3 : new target (preserved for callee if needed, and caller)
// -- x1 : target function (preserved for callee if needed, and caller)
- // -- feedback vector (preserved for caller if needed)
// -----------------------------------
- DCHECK(!AreAliased(feedback_vector, x1, x3, scratch1, scratch2));
-
- Label optimized_code_slot_is_weak_ref, fallthrough;
+ DCHECK(!AreAliased(x1, x3, optimized_code_entry, scratch));
Register closure = x1;
- Register optimized_code_entry = scratch1;
-
- __ LoadAnyTaggedField(
- optimized_code_entry,
- FieldMemOperand(feedback_vector,
- FeedbackVector::kOptimizedCodeWeakOrSmiOffset));
-
- // Check if the code entry is a Smi. If yes, we interpret it as an
- // optimisation marker. Otherwise, interpret is at a weak reference to a code
- // object.
- __ JumpIfNotSmi(optimized_code_entry, &optimized_code_slot_is_weak_ref);
-
- {
- // Optimized code slot is a Smi optimization marker.
-
- // Fall through if no optimization trigger.
- __ CompareAndBranch(optimized_code_entry,
- Operand(Smi::FromEnum(OptimizationMarker::kNone)), eq,
- &fallthrough);
-
- // TODO(v8:8394): The logging of first execution will break if
- // feedback vectors are not allocated. We need to find a different way of
- // logging these events if required.
- TailCallRuntimeIfMarkerEquals(masm, optimized_code_entry,
- OptimizationMarker::kLogFirstExecution,
- Runtime::kFunctionFirstExecution);
- TailCallRuntimeIfMarkerEquals(masm, optimized_code_entry,
- OptimizationMarker::kCompileOptimized,
- Runtime::kCompileOptimized_NotConcurrent);
- TailCallRuntimeIfMarkerEquals(
- masm, optimized_code_entry,
- OptimizationMarker::kCompileOptimizedConcurrent,
- Runtime::kCompileOptimized_Concurrent);
-
- {
- // Otherwise, the marker is InOptimizationQueue, so fall through hoping
- // that an interrupt will eventually update the slot with optimized code.
- if (FLAG_debug_code) {
- __ Cmp(
- optimized_code_entry,
- Operand(Smi::FromEnum(OptimizationMarker::kInOptimizationQueue)));
- __ Assert(eq, AbortReason::kExpectedOptimizationSentinel);
- }
- __ B(&fallthrough);
- }
- }
- {
- // Optimized code slot is a weak reference.
- __ bind(&optimized_code_slot_is_weak_ref);
-
- __ LoadWeakValue(optimized_code_entry, optimized_code_entry, &fallthrough);
+ // Check if the optimized code is marked for deopt. If it is, call the
+ // runtime to clear it.
+ Label found_deoptimized_code;
+ __ LoadTaggedPointerField(
+ scratch,
+ FieldMemOperand(optimized_code_entry, Code::kCodeDataContainerOffset));
+ __ Ldr(scratch.W(),
+ FieldMemOperand(scratch, CodeDataContainer::kKindSpecificFlagsOffset));
+ __ Tbnz(scratch.W(), Code::kMarkedForDeoptimizationBit,
+ &found_deoptimized_code);
+
+ // Optimized code is good, get it into the closure and link the closure into
+ // the optimized functions list, then tail call the optimized code.
+ ReplaceClosureCodeWithOptimizedCode(masm, optimized_code_entry, closure);
+ static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
+ __ LoadCodeObjectEntry(x2, optimized_code_entry);
+ __ Jump(x2);
- // Check if the optimized code is marked for deopt. If it is, call the
- // runtime to clear it.
- Label found_deoptimized_code;
- __ LoadTaggedPointerField(
- scratch2,
- FieldMemOperand(optimized_code_entry, Code::kCodeDataContainerOffset));
- __ Ldr(
- scratch2.W(),
- FieldMemOperand(scratch2, CodeDataContainer::kKindSpecificFlagsOffset));
- __ Tbnz(scratch2.W(), Code::kMarkedForDeoptimizationBit,
- &found_deoptimized_code);
-
- // Optimized code is good, get it into the closure and link the closure into
- // the optimized functions list, then tail call the optimized code.
- // The feedback vector is no longer used, so re-use it as a scratch
- // register.
- ReplaceClosureCodeWithOptimizedCode(masm, optimized_code_entry, closure);
- static_assert(kJavaScriptCallCodeStartRegister == x2, "ABI mismatch");
- __ LoadCodeObjectEntry(x2, optimized_code_entry);
- __ Jump(x2);
+ // Optimized code slot contains deoptimized code, evict it and re-enter the
+ // closure's code.
+ __ bind(&found_deoptimized_code);
+ GenerateTailCallToReturnedCode(masm, Runtime::kEvictOptimizedCodeSlot);
+}
- // Optimized code slot contains deoptimized code, evict it and re-enter the
- // closure's code.
- __ bind(&found_deoptimized_code);
- GenerateTailCallToReturnedCode(masm, Runtime::kEvictOptimizedCodeSlot);
+static void MaybeOptimizeCode(MacroAssembler* masm, Register feedback_vector,
+ Register optimization_marker) {
+ // ----------- S t a t e -------------
+ // -- x3 : new target (preserved for callee if needed, and caller)
+ // -- x1 : target function (preserved for callee if needed, and caller)
+ // -- feedback vector (preserved for caller if needed)
+ // -- optimization_marker : a Smi containing a non-zero optimization marker.
+ // -----------------------------------
+ DCHECK(!AreAliased(feedback_vector, x1, x3, optimization_marker));
+
+ // TODO(v8:8394): The logging of first execution will break if
+ // feedback vectors are not allocated. We need to find a different way of
+ // logging these events if required.
+ TailCallRuntimeIfMarkerEquals(masm, optimization_marker,
+ OptimizationMarker::kLogFirstExecution,
+ Runtime::kFunctionFirstExecution);
+ TailCallRuntimeIfMarkerEquals(masm, optimization_marker,
+ OptimizationMarker::kCompileOptimized,
+ Runtime::kCompileOptimized_NotConcurrent);
+ TailCallRuntimeIfMarkerEquals(masm, optimization_marker,
+ OptimizationMarker::kCompileOptimizedConcurrent,
+ Runtime::kCompileOptimized_Concurrent);
+
+ // Otherwise, the marker is InOptimizationQueue, so fall through hoping
+ // that an interrupt will eventually update the slot with optimized code.
+ if (FLAG_debug_code) {
+ __ CmpTagged(
+ optimization_marker,
+ Operand(Smi::FromEnum(OptimizationMarker::kInOptimizationQueue)));
+ __ Assert(eq, AbortReason::kExpectedOptimizationSentinel);
}
-
- // Fall-through if the optimized code cell is clear and there is no
- // optimization marker.
- __ bind(&fallthrough);
}
// Advance the current bytecode offset. This simulates what all bytecode
@@ -1129,19 +1099,19 @@ static void AdvanceBytecodeOffsetOrReturn(MacroAssembler* masm,
__ Cmp(bytecode, Operand(0x3));
__ B(hi, &process_bytecode);
__ Tst(bytecode, Operand(0x1));
- __ B(ne, &extra_wide);
-
- // Load the next bytecode and update table to the wide scaled table.
+ // The code to load the next bytecode is common to both wide and extra wide.
+ // We can hoist them up here since they do not modify the flags after Tst.
__ Add(bytecode_offset, bytecode_offset, Operand(1));
__ Ldrb(bytecode, MemOperand(bytecode_array, bytecode_offset));
+ __ B(ne, &extra_wide);
+
+ // Update table to the wide scaled table.
__ Add(bytecode_size_table, bytecode_size_table,
Operand(kIntSize * interpreter::Bytecodes::kBytecodeCount));
__ B(&process_bytecode);
__ Bind(&extra_wide);
- // Load the next bytecode and update table to the extra wide scaled table.
- __ Add(bytecode_offset, bytecode_offset, Operand(1));
- __ Ldrb(bytecode, MemOperand(bytecode_array, bytecode_offset));
+ // Update table to the extra wide scaled table.
__ Add(bytecode_size_table, bytecode_size_table,
Operand(2 * kIntSize * interpreter::Bytecodes::kBytecodeCount));
@@ -1211,7 +1181,20 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
// Read off the optimized code slot in the feedback vector, and if there
// is optimized code or an optimization marker, call that instead.
- MaybeTailCallOptimizedCodeSlot(masm, feedback_vector, x7, x4);
+ Register optimized_code_entry = x7;
+ __ LoadAnyTaggedField(
+ optimized_code_entry,
+ FieldMemOperand(feedback_vector,
+ FeedbackVector::kOptimizedCodeWeakOrSmiOffset));
+
+ // Check if the optimized code slot is not empty.
+ Label optimized_code_slot_not_empty;
+ __ CompareTaggedAndBranch(optimized_code_entry,
+ Operand(Smi::FromEnum(OptimizationMarker::kNone)),
+ ne, &optimized_code_slot_not_empty);
+
+ Label not_optimized;
+ __ bind(&not_optimized);
// Increment invocation count for the function.
// MaybeTailCallOptimizedCodeSlot preserves feedback_vector, so safe to reuse
@@ -1248,13 +1231,13 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
__ Push(kInterpreterBytecodeArrayRegister, x0);
// Allocate the local and temporary register file on the stack.
+ Label stack_overflow;
{
// Load frame size from the BytecodeArray object.
__ Ldr(w11, FieldMemOperand(kInterpreterBytecodeArrayRegister,
BytecodeArray::kFrameSizeOffset));
// Do a stack check to ensure we don't go over the limit.
- Label ok;
__ Sub(x10, sp, Operand(x11));
{
UseScratchRegisterScope temps(masm);
@@ -1262,21 +1245,19 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
LoadRealStackLimit(masm, scratch);
__ Cmp(x10, scratch);
}
- __ B(hs, &ok);
- __ CallRuntime(Runtime::kThrowStackOverflow);
- __ Bind(&ok);
+ __ B(lo, &stack_overflow);
// If ok, push undefined as the initial value for all register file entries.
// Note: there should always be at least one stack slot for the return
// register in the register file.
Label loop_header;
- __ LoadRoot(x10, RootIndex::kUndefinedValue);
+ __ LoadRoot(kInterpreterAccumulatorRegister, RootIndex::kUndefinedValue);
__ Lsr(x11, x11, kSystemPointerSizeLog2);
// Round up the number of registers to a multiple of 2, to align the stack
// to 16 bytes.
__ Add(x11, x11, 1);
__ Bic(x11, x11, 1);
- __ PushMultipleTimes(x10, x11);
+ __ PushMultipleTimes(kInterpreterAccumulatorRegister, x11);
__ Bind(&loop_header);
}
@@ -1291,8 +1272,7 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
__ Str(x3, MemOperand(fp, x10, LSL, kSystemPointerSizeLog2));
__ Bind(&no_incoming_new_target_or_generator_register);
- // Load accumulator with undefined.
- __ LoadRoot(kInterpreterAccumulatorRegister, RootIndex::kUndefinedValue);
+ // The accumulator is already loaded with undefined.
// Load the dispatch table into a register and dispatch to the bytecode
// handler at the current bytecode offset.
@@ -1315,9 +1295,8 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
// Get bytecode array and bytecode offset from the stack frame.
__ Ldr(kInterpreterBytecodeArrayRegister,
MemOperand(fp, InterpreterFrameConstants::kBytecodeArrayFromFp));
- __ Ldr(kInterpreterBytecodeOffsetRegister,
- MemOperand(fp, InterpreterFrameConstants::kBytecodeOffsetFromFp));
- __ SmiUntag(kInterpreterBytecodeOffsetRegister);
+ __ SmiUntag(kInterpreterBytecodeOffsetRegister,
+ MemOperand(fp, InterpreterFrameConstants::kBytecodeOffsetFromFp));
// Either return, or advance to the next bytecode and dispatch.
Label do_return;
@@ -1333,9 +1312,28 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
LeaveInterpreterFrame(masm, x2);
__ Ret();
+ __ bind(&optimized_code_slot_not_empty);
+ Label maybe_has_optimized_code;
+ // Check if optimized code marker is actually a weak reference to the
+ // optimized code as opposed to an optimization marker.
+ __ JumpIfNotSmi(optimized_code_entry, &maybe_has_optimized_code);
+ MaybeOptimizeCode(masm, feedback_vector, optimized_code_entry);
+ // Fall through if there's no runnable optimized code.
+ __ jmp(&not_optimized);
+
+ __ bind(&maybe_has_optimized_code);
+ // Load code entry from the weak reference, if it was cleared, resume
+ // execution of unoptimized code.
+ __ LoadWeakValue(optimized_code_entry, optimized_code_entry, &not_optimized);
+ TailCallOptimizedCodeSlot(masm, optimized_code_entry, x4);
+
__ bind(&compile_lazy);
GenerateTailCallToReturnedCode(masm, Runtime::kCompileLazy);
__ Unreachable(); // Should not return.
+
+ __ bind(&stack_overflow);
+ __ CallRuntime(Runtime::kThrowStackOverflow);
+ __ Unreachable(); // Should not return.
}
static void Generate_InterpreterPushArgs(MacroAssembler* masm,
@@ -1543,9 +1541,8 @@ static void Generate_InterpreterEnterBytecode(MacroAssembler* masm) {
}
// Get the target bytecode offset from the frame.
- __ Ldr(kInterpreterBytecodeOffsetRegister,
- MemOperand(fp, InterpreterFrameConstants::kBytecodeOffsetFromFp));
- __ SmiUntag(kInterpreterBytecodeOffsetRegister);
+ __ SmiUntag(kInterpreterBytecodeOffsetRegister,
+ MemOperand(fp, InterpreterFrameConstants::kBytecodeOffsetFromFp));
// Dispatch to the target bytecode.
__ Ldrb(x23, MemOperand(kInterpreterBytecodeArrayRegister,
@@ -1560,9 +1557,8 @@ void Builtins::Generate_InterpreterEnterBytecodeAdvance(MacroAssembler* masm) {
// Get bytecode array and bytecode offset from the stack frame.
__ ldr(kInterpreterBytecodeArrayRegister,
MemOperand(fp, InterpreterFrameConstants::kBytecodeArrayFromFp));
- __ ldr(kInterpreterBytecodeOffsetRegister,
- MemOperand(fp, InterpreterFrameConstants::kBytecodeOffsetFromFp));
- __ SmiUntag(kInterpreterBytecodeOffsetRegister);
+ __ SmiUntag(kInterpreterBytecodeOffsetRegister,
+ MemOperand(fp, InterpreterFrameConstants::kBytecodeOffsetFromFp));
// Load the current bytecode.
__ Ldrb(x1, MemOperand(kInterpreterBytecodeArrayRegister,
@@ -1633,7 +1629,7 @@ void Builtins::Generate_InstantiateAsmJs(MacroAssembler* masm) {
// Set flags for determining the value of smi-tagged argc.
// lt => 1, eq => 2, gt => 3.
- __ Cmp(argc, Smi::FromInt(2));
+ __ CmpTagged(argc, Smi::FromInt(2));
__ B(gt, &three_args);
// One or two arguments.
@@ -1769,20 +1765,14 @@ void Builtins::Generate_NotifyDeoptimized(MacroAssembler* masm) {
}
void Builtins::Generate_InterpreterOnStackReplacement(MacroAssembler* masm) {
- // Lookup the function in the JavaScript frame.
- __ Ldr(x0, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
- __ Ldr(x0, MemOperand(x0, JavaScriptFrameConstants::kFunctionOffset));
-
{
FrameScope scope(masm, StackFrame::INTERNAL);
- // Pass function as argument.
- __ PushArgument(x0);
__ CallRuntime(Runtime::kCompileForOnStackReplacement);
}
// If the code object is null, just return to the caller.
Label skip;
- __ CompareAndBranch(x0, Smi::zero(), ne, &skip);
+ __ CompareTaggedAndBranch(x0, Smi::zero(), ne, &skip);
__ Ret();
__ Bind(&skip);
@@ -1878,8 +1868,8 @@ void Builtins::Generate_FunctionPrototypeApply(MacroAssembler* masm) {
// 3. Tail call with no arguments if argArray is null or undefined.
Label no_arguments;
- __ Cmp(arg_array, null_value);
- __ Ccmp(arg_array, undefined_value, ZFlag, ne);
+ __ CmpTagged(arg_array, null_value);
+ __ CcmpTagged(arg_array, undefined_value, ZFlag, ne);
__ B(eq, &no_arguments);
// 4a. Apply the receiver to the given argArray.
@@ -2261,7 +2251,7 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
__ Bind(&loop);
__ Sub(len, len, 1);
__ LoadAnyTaggedField(scratch, MemOperand(src, kTaggedSize, PostIndex));
- __ Cmp(scratch, the_hole_value);
+ __ CmpTagged(scratch, the_hole_value);
__ Csel(scratch, scratch, undefined_value, ne);
__ Poke(scratch, Operand(len, LSL, kSystemPointerSizeLog2));
__ Cbnz(len, &loop);
@@ -2319,7 +2309,7 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
__ Ldr(args_fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
__ Ldr(x4, MemOperand(args_fp,
CommonFrameConstants::kContextOrFrameTypeOffset));
- __ Cmp(x4, StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR));
+ __ CmpTagged(x4, StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR));
__ B(eq, &arguments_adaptor);
{
__ Ldr(scratch,
@@ -2626,7 +2616,7 @@ void Builtins::Generate_Call(MacroAssembler* masm, ConvertReceiverMode mode) {
// -- x1 : the target to call (can be any Object).
// -----------------------------------
- Label non_callable, non_function, non_smi;
+ Label non_callable, non_smi;
__ JumpIfSmi(x1, &non_callable);
__ Bind(&non_smi);
__ CompareObjectType(x1, x4, x5, JS_FUNCTION_TYPE);
@@ -2642,12 +2632,10 @@ void Builtins::Generate_Call(MacroAssembler* masm, ConvertReceiverMode mode) {
// Check if target is a proxy and call CallProxy external builtin
__ Cmp(x5, JS_PROXY_TYPE);
- __ B(ne, &non_function);
- __ Jump(BUILTIN_CODE(masm->isolate(), CallProxy), RelocInfo::CODE_TARGET);
+ __ Jump(BUILTIN_CODE(masm->isolate(), CallProxy), RelocInfo::CODE_TARGET, eq);
// 2. Call to something else, which might have a [[Call]] internal method (if
// not we raise an exception).
- __ Bind(&non_function);
// Overwrite the original receiver with the (original) target.
__ Poke(x1, Operand(x0, LSL, kXRegSizeLog2));
// Let the "call_as_function_delegate" take care of the rest.
@@ -2712,7 +2700,7 @@ void Builtins::Generate_ConstructBoundFunction(MacroAssembler* masm) {
// Patch new.target to [[BoundTargetFunction]] if new.target equals target.
{
Label done;
- __ Cmp(x1, x3);
+ __ CmpTagged(x1, x3);
__ B(ne, &done);
__ LoadTaggedPointerField(
x3, FieldMemOperand(x1, JSBoundFunction::kBoundTargetFunctionOffset));
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-01 01:45:08 +0000