diff options
Diffstat (limited to 'deps/v8/src/builtins/x64/builtins-x64.cc')
-rw-r--r-- | deps/v8/src/builtins/x64/builtins-x64.cc | 279 |
1 files changed, 135 insertions, 144 deletions
diff --git a/deps/v8/src/builtins/x64/builtins-x64.cc b/deps/v8/src/builtins/x64/builtins-x64.cc index b6b407fb33..9679237ff8 100644 --- a/deps/v8/src/builtins/x64/builtins-x64.cc +++ b/deps/v8/src/builtins/x64/builtins-x64.cc @@ -5,8 +5,9 @@ #if V8_TARGET_ARCH_X64 #include "src/api/api-arguments.h" -#include "src/base/adapters.h" +#include "src/base/iterator.h" #include "src/codegen/code-factory.h" +#include "src/codegen/x64/assembler-x64.h" #include "src/deoptimizer/deoptimizer.h" #include "src/execution/frame-constants.h" #include "src/execution/frames.h" @@ -401,13 +402,13 @@ void Generate_JSEntryVariant(MacroAssembler* masm, StackFrame::Type type, __ pushq(r13); __ pushq(r14); __ pushq(r15); -#ifdef _WIN64 +#ifdef V8_TARGET_OS_WIN __ pushq(rdi); // Only callee save in Win64 ABI, argument in AMD64 ABI. __ pushq(rsi); // Only callee save in Win64 ABI, argument in AMD64 ABI. #endif __ pushq(rbx); -#ifdef _WIN64 +#ifdef V8_TARGET_OS_WIN // On Win64 XMM6-XMM15 are callee-save. __ AllocateStackSpace(EntryFrameConstants::kXMMRegistersBlockSize); __ movdqu(Operand(rsp, EntryFrameConstants::kXMMRegisterSize * 0), xmm6); @@ -507,7 +508,7 @@ void Generate_JSEntryVariant(MacroAssembler* masm, StackFrame::Type type, } // Restore callee-saved registers (X64 conventions). -#ifdef _WIN64 +#ifdef V8_TARGET_OS_WIN // On Win64 XMM6-XMM15 are callee-save __ movdqu(xmm6, Operand(rsp, EntryFrameConstants::kXMMRegisterSize * 0)); __ movdqu(xmm7, Operand(rsp, EntryFrameConstants::kXMMRegisterSize * 1)); @@ -523,7 +524,7 @@ void Generate_JSEntryVariant(MacroAssembler* masm, StackFrame::Type type, #endif __ popq(rbx); -#ifdef _WIN64 +#ifdef V8_TARGET_OS_WIN // Callee save on in Win64 ABI, arguments/volatile in AMD64 ABI. __ popq(rsi); __ popq(rdi); @@ -611,17 +612,17 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm, __ Push(rdi); __ Push(arg_reg_4); -#ifdef _WIN64 +#ifdef V8_TARGET_OS_WIN // Load the previous frame pointer to access C arguments on stack __ movq(kScratchRegister, Operand(rbp, 0)); // Load the number of arguments and setup pointer to the arguments. __ movq(rax, Operand(kScratchRegister, EntryFrameConstants::kArgcOffset)); __ movq(rbx, Operand(kScratchRegister, EntryFrameConstants::kArgvOffset)); -#else // _WIN64 +#else // V8_TARGET_OS_WIN // Load the number of arguments and setup pointer to the arguments. __ movq(rax, r8); __ movq(rbx, r9); -#endif // _WIN64 +#endif // V8_TARGET_OS_WIN // Current stack contents: // [rsp + 2 * kSystemPointerSize ... ] : Internal frame @@ -851,10 +852,11 @@ void Builtins::Generate_ResumeGeneratorTrampoline(MacroAssembler* masm) { // TODO(juliana): if we remove the code below then we don't need all // the parameters. -static void ReplaceClosureCodeWithOptimizedCode( - MacroAssembler* masm, Register optimized_code, Register closure, - Register scratch1, Register scratch2, Register scratch3) { - +static void ReplaceClosureCodeWithOptimizedCode(MacroAssembler* masm, + Register optimized_code, + Register closure, + Register scratch1, + Register scratch2) { // Store the optimized code in the closure. __ StoreTaggedField(FieldOperand(closure, JSFunction::kCodeOffset), optimized_code); @@ -895,104 +897,71 @@ static void TailCallRuntimeIfMarkerEquals(MacroAssembler* masm, __ bind(&no_match); } -static void MaybeTailCallOptimizedCodeSlot(MacroAssembler* masm, - Register feedback_vector, - Register scratch1, Register scratch2, - Register scratch3) { +static void MaybeOptimizeCode(MacroAssembler* masm, Register feedback_vector, + Register optimization_marker) { // ----------- S t a t e ------------- // -- rdx : new target (preserved for callee if needed, and caller) // -- rdi : target function (preserved for callee if needed, and caller) // -- feedback vector (preserved for caller if needed) + // -- optimization_marker : a Smi containing a non-zero optimization marker. // ----------------------------------- - DCHECK(!AreAliased(feedback_vector, rdx, rdi, scratch1, scratch2, scratch3)); - - Label optimized_code_slot_is_weak_ref, fallthrough; - - Register closure = rdi; - Register optimized_code_entry = scratch1; - Register decompr_scratch = COMPRESS_POINTERS_BOOL ? scratch2 : no_reg; - - __ LoadAnyTaggedField( - optimized_code_entry, - FieldOperand(feedback_vector, - FeedbackVector::kOptimizedCodeWeakOrSmiOffset), - decompr_scratch); - - // Check if the code entry is a Smi. If yes, we interpret it as an - // optimisation marker. Otherwise, interpret it as a weak reference to a code - // object. - __ JumpIfNotSmi(optimized_code_entry, &optimized_code_slot_is_weak_ref); - - { - // Optimized code slot is a Smi optimization marker. - - // Fall through if no optimization trigger. - __ SmiCompare(optimized_code_entry, - Smi::FromEnum(OptimizationMarker::kNone)); - __ j(equal, &fallthrough); - - // TODO(v8:8394): The logging of first execution will break if - // feedback vectors are not allocated. We need to find a different way of - // logging these events if required. - TailCallRuntimeIfMarkerEquals(masm, optimized_code_entry, - OptimizationMarker::kLogFirstExecution, - Runtime::kFunctionFirstExecution); - TailCallRuntimeIfMarkerEquals(masm, optimized_code_entry, - OptimizationMarker::kCompileOptimized, - Runtime::kCompileOptimized_NotConcurrent); - TailCallRuntimeIfMarkerEquals( - masm, optimized_code_entry, - OptimizationMarker::kCompileOptimizedConcurrent, - Runtime::kCompileOptimized_Concurrent); - { - // Otherwise, the marker is InOptimizationQueue, so fall through hoping - // that an interrupt will eventually update the slot with optimized code. - if (FLAG_debug_code) { - __ SmiCompare(optimized_code_entry, - Smi::FromEnum(OptimizationMarker::kInOptimizationQueue)); - __ Assert(equal, AbortReason::kExpectedOptimizationSentinel); - } - __ jmp(&fallthrough); - } + DCHECK(!AreAliased(feedback_vector, rdx, rdi, optimization_marker)); + + // TODO(v8:8394): The logging of first execution will break if + // feedback vectors are not allocated. We need to find a different way of + // logging these events if required. + TailCallRuntimeIfMarkerEquals(masm, optimization_marker, + OptimizationMarker::kLogFirstExecution, + Runtime::kFunctionFirstExecution); + TailCallRuntimeIfMarkerEquals(masm, optimization_marker, + OptimizationMarker::kCompileOptimized, + Runtime::kCompileOptimized_NotConcurrent); + TailCallRuntimeIfMarkerEquals(masm, optimization_marker, + OptimizationMarker::kCompileOptimizedConcurrent, + Runtime::kCompileOptimized_Concurrent); + + // Otherwise, the marker is InOptimizationQueue, so fall through hoping + // that an interrupt will eventually update the slot with optimized code. + if (FLAG_debug_code) { + __ SmiCompare(optimization_marker, + Smi::FromEnum(OptimizationMarker::kInOptimizationQueue)); + __ Assert(equal, AbortReason::kExpectedOptimizationSentinel); } +} - { - // Optimized code slot is a weak reference. - __ bind(&optimized_code_slot_is_weak_ref); - - __ LoadWeakValue(optimized_code_entry, &fallthrough); +static void TailCallOptimizedCodeSlot(MacroAssembler* masm, + Register optimized_code_entry, + Register scratch1, Register scratch2) { + // ----------- S t a t e ------------- + // -- rdx : new target (preserved for callee if needed, and caller) + // -- rdi : target function (preserved for callee if needed, and caller) + // ----------------------------------- - // Check if the optimized code is marked for deopt. If it is, call the - // runtime to clear it. - Label found_deoptimized_code; - __ LoadTaggedPointerField( - scratch2, - FieldOperand(optimized_code_entry, Code::kCodeDataContainerOffset)); - __ testl( - FieldOperand(scratch2, CodeDataContainer::kKindSpecificFlagsOffset), - Immediate(1 << Code::kMarkedForDeoptimizationBit)); - __ j(not_zero, &found_deoptimized_code); - - // Optimized code is good, get it into the closure and link the closure into - // the optimized functions list, then tail call the optimized code. - // The feedback vector is no longer used, so re-use it as a scratch - // register. - ReplaceClosureCodeWithOptimizedCode(masm, optimized_code_entry, closure, - scratch2, scratch3, feedback_vector); - static_assert(kJavaScriptCallCodeStartRegister == rcx, "ABI mismatch"); - __ Move(rcx, optimized_code_entry); - __ JumpCodeObject(rcx); + Register closure = rdi; - // Optimized code slot contains deoptimized code, evict it and re-enter the - // closure's code. - __ bind(&found_deoptimized_code); - GenerateTailCallToReturnedCode(masm, Runtime::kEvictOptimizedCodeSlot); - } + // Check if the optimized code is marked for deopt. If it is, call the + // runtime to clear it. + Label found_deoptimized_code; + __ LoadTaggedPointerField( + scratch1, + FieldOperand(optimized_code_entry, Code::kCodeDataContainerOffset)); + __ testl(FieldOperand(scratch1, CodeDataContainer::kKindSpecificFlagsOffset), + Immediate(1 << Code::kMarkedForDeoptimizationBit)); + __ j(not_zero, &found_deoptimized_code); + + // Optimized code is good, get it into the closure and link the closure into + // the optimized functions list, then tail call the optimized code. + ReplaceClosureCodeWithOptimizedCode(masm, optimized_code_entry, closure, + scratch1, scratch2); + static_assert(kJavaScriptCallCodeStartRegister == rcx, "ABI mismatch"); + __ Move(rcx, optimized_code_entry); + __ JumpCodeObject(rcx); - // Fall-through if the optimized code cell is clear and there is no - // optimization marker. - __ bind(&fallthrough); + // Optimized code slot contains deoptimized code, evict it and re-enter the + // closure's code. + __ bind(&found_deoptimized_code); + GenerateTailCallToReturnedCode(masm, Runtime::kEvictOptimizedCodeSlot); } // Advance the current bytecode offset. This simulates what all bytecode @@ -1019,20 +988,21 @@ static void AdvanceBytecodeOffsetOrReturn(MacroAssembler* masm, static_cast<int>(interpreter::Bytecode::kDebugBreakExtraWide)); __ cmpb(bytecode, Immediate(0x3)); __ j(above, &process_bytecode, Label::kNear); + // The code to load the next bytecode is common to both wide and extra wide. + // We can hoist them up here. incl has to happen before testb since it + // modifies the ZF flag. + __ incl(bytecode_offset); __ testb(bytecode, Immediate(0x1)); + __ movzxbq(bytecode, Operand(bytecode_array, bytecode_offset, times_1, 0)); __ j(not_equal, &extra_wide, Label::kNear); - // Load the next bytecode and update table to the wide scaled table. - __ incl(bytecode_offset); - __ movzxbq(bytecode, Operand(bytecode_array, bytecode_offset, times_1, 0)); + // Update table to the wide scaled table. __ addq(bytecode_size_table, Immediate(kIntSize * interpreter::Bytecodes::kBytecodeCount)); __ jmp(&process_bytecode, Label::kNear); __ bind(&extra_wide); - // Load the next bytecode and update table to the extra wide scaled table. - __ incl(bytecode_offset); - __ movzxbq(bytecode, Operand(bytecode_array, bytecode_offset, times_1, 0)); + // Update table to the extra wide scaled table. __ addq(bytecode_size_table, Immediate(2 * kIntSize * interpreter::Bytecodes::kBytecodeCount)); @@ -1101,7 +1071,23 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) { // Read off the optimized code slot in the feedback vector, and if there // is optimized code or an optimization marker, call that instead. - MaybeTailCallOptimizedCodeSlot(masm, feedback_vector, rcx, r11, r15); + + Register optimized_code_entry = rcx; + Register decompr_scratch = COMPRESS_POINTERS_BOOL ? r11 : no_reg; + + __ LoadAnyTaggedField( + optimized_code_entry, + FieldOperand(feedback_vector, + FeedbackVector::kOptimizedCodeWeakOrSmiOffset), + decompr_scratch); + + // Check if the optimized code slot is not empty. + Label optimized_code_slot_not_empty; + __ Cmp(optimized_code_entry, Smi::FromEnum(OptimizationMarker::kNone)); + __ j(not_equal, &optimized_code_slot_not_empty); + + Label not_optimized; + __ bind(¬_optimized); // Increment invocation count for the function. __ incl( @@ -1137,28 +1123,26 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) { __ Push(rcx); // Allocate the local and temporary register file on the stack. + Label stack_overflow; { // Load frame size from the BytecodeArray object. __ movl(rcx, FieldOperand(kInterpreterBytecodeArrayRegister, BytecodeArray::kFrameSizeOffset)); // Do a stack check to ensure we don't go over the limit. - Label ok; __ movq(rax, rsp); __ subq(rax, rcx); __ cmpq(rax, RealStackLimitAsOperand(masm)); - __ j(above_equal, &ok, Label::kNear); - __ CallRuntime(Runtime::kThrowStackOverflow); - __ bind(&ok); + __ j(below, &stack_overflow); // If ok, push undefined as the initial value for all register file entries. Label loop_header; Label loop_check; - __ LoadRoot(rax, RootIndex::kUndefinedValue); + __ LoadRoot(kInterpreterAccumulatorRegister, RootIndex::kUndefinedValue); __ j(always, &loop_check, Label::kNear); __ bind(&loop_header); // TODO(rmcilroy): Consider doing more than one push per loop iteration. - __ Push(rax); + __ Push(kInterpreterAccumulatorRegister); // Continue loop if not done. __ bind(&loop_check); __ subq(rcx, Immediate(kSystemPointerSize)); @@ -1169,16 +1153,15 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) { // register, initialize it with incoming value which was passed in rdx. Label no_incoming_new_target_or_generator_register; __ movsxlq( - rax, + rcx, FieldOperand(kInterpreterBytecodeArrayRegister, BytecodeArray::kIncomingNewTargetOrGeneratorRegisterOffset)); - __ testl(rax, rax); + __ testl(rcx, rcx); __ j(zero, &no_incoming_new_target_or_generator_register, Label::kNear); - __ movq(Operand(rbp, rax, times_system_pointer_size, 0), rdx); + __ movq(Operand(rbp, rcx, times_system_pointer_size, 0), rdx); __ bind(&no_incoming_new_target_or_generator_register); - // Load accumulator with undefined. - __ LoadRoot(kInterpreterAccumulatorRegister, RootIndex::kUndefinedValue); + // The accumulator is already loaded with undefined. // Load the dispatch table into a register and dispatch to the bytecode // handler at the current bytecode offset. @@ -1201,10 +1184,8 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) { // Get bytecode array and bytecode offset from the stack frame. __ movq(kInterpreterBytecodeArrayRegister, Operand(rbp, InterpreterFrameConstants::kBytecodeArrayFromFp)); - __ movq(kInterpreterBytecodeOffsetRegister, - Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp)); __ SmiUntag(kInterpreterBytecodeOffsetRegister, - kInterpreterBytecodeOffsetRegister); + Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp)); // Either return, or advance to the next bytecode and dispatch. Label do_return; @@ -1223,6 +1204,25 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) { __ bind(&compile_lazy); GenerateTailCallToReturnedCode(masm, Runtime::kCompileLazy); __ int3(); // Should not return. + + __ bind(&optimized_code_slot_not_empty); + Label maybe_has_optimized_code; + // Check if optimized code marker is actually a weak reference to the + // optimized code as opposed to an optimization marker. + __ JumpIfNotSmi(optimized_code_entry, &maybe_has_optimized_code); + MaybeOptimizeCode(masm, feedback_vector, optimized_code_entry); + // Fall through if there's no runnable optimized code. + __ jmp(¬_optimized); + + __ bind(&maybe_has_optimized_code); + // Load code entry from the weak reference, if it was cleared, resume + // execution of unoptimized code. + __ LoadWeakValue(optimized_code_entry, ¬_optimized); + TailCallOptimizedCodeSlot(masm, optimized_code_entry, r11, r15); + + __ bind(&stack_overflow); + __ CallRuntime(Runtime::kThrowStackOverflow); + __ int3(); // Should not return. } static void Generate_InterpreterPushArgs(MacroAssembler* masm, @@ -1425,10 +1425,8 @@ static void Generate_InterpreterEnterBytecode(MacroAssembler* masm) { } // Get the target bytecode offset from the frame. - __ movq(kInterpreterBytecodeOffsetRegister, - Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp)); __ SmiUntag(kInterpreterBytecodeOffsetRegister, - kInterpreterBytecodeOffsetRegister); + Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp)); // Dispatch to the target bytecode. __ movzxbq(r11, Operand(kInterpreterBytecodeArrayRegister, @@ -1443,10 +1441,8 @@ void Builtins::Generate_InterpreterEnterBytecodeAdvance(MacroAssembler* masm) { // Get bytecode array and bytecode offset from the stack frame. __ movq(kInterpreterBytecodeArrayRegister, Operand(rbp, InterpreterFrameConstants::kBytecodeArrayFromFp)); - __ movq(kInterpreterBytecodeOffsetRegister, - Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp)); __ SmiUntag(kInterpreterBytecodeOffsetRegister, - kInterpreterBytecodeOffsetRegister); + Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp)); // Load the current bytecode. __ movzxbq(rbx, Operand(kInterpreterBytecodeArrayRegister, @@ -1459,8 +1455,9 @@ void Builtins::Generate_InterpreterEnterBytecodeAdvance(MacroAssembler* masm) { &if_return); // Convert new bytecode offset to a Smi and save in the stackframe. - __ SmiTag(rbx, kInterpreterBytecodeOffsetRegister); - __ movq(Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp), rbx); + __ SmiTag(kInterpreterBytecodeOffsetRegister); + __ movq(Operand(rbp, InterpreterFrameConstants::kBytecodeOffsetFromFp), + kInterpreterBytecodeOffsetRegister); Generate_InterpreterEnterBytecode(masm); @@ -1485,7 +1482,7 @@ void Builtins::Generate_InstantiateAsmJs(MacroAssembler* masm) { // Preserve argument count for later compare. __ movq(rcx, rax); // Push the number of arguments to the callee. - __ SmiTag(rax, rax); + __ SmiTag(rax); __ Push(rax); // Push a copy of the target function and the new target. __ Push(rdi); @@ -1522,7 +1519,7 @@ void Builtins::Generate_InstantiateAsmJs(MacroAssembler* masm) { __ Drop(2); __ Pop(rcx); - __ SmiUntag(rcx, rcx); + __ SmiUntag(rcx); scope.GenerateLeaveFrame(); __ PopReturnAddressTo(rbx); @@ -1536,7 +1533,7 @@ void Builtins::Generate_InstantiateAsmJs(MacroAssembler* masm) { __ Pop(rdx); __ Pop(rdi); __ Pop(rax); - __ SmiUntag(rax, rax); + __ SmiUntag(rax); } // On failure, tail call back to regular js by re-calling the function // which has be reset to the compile lazy builtin. @@ -1563,7 +1560,7 @@ void Generate_ContinueToBuiltinHelper(MacroAssembler* masm, int code = config->GetAllocatableGeneralCode(i); __ popq(Register::from_code(code)); if (java_script_builtin && code == kJavaScriptCallArgCountRegister.code()) { - __ SmiUntag(Register::from_code(code), Register::from_code(code)); + __ SmiUntag(Register::from_code(code)); } } __ movq( @@ -2274,7 +2271,7 @@ void Builtins::Generate_CallFunction(MacroAssembler* masm, // TODO(bmeurer): Inline the allocation here to avoid building the frame // in the fast case? (fall back to AllocateInNewSpace?) FrameScope scope(masm, StackFrame::INTERNAL); - __ SmiTag(rax, rax); + __ SmiTag(rax); __ Push(rax); __ Push(rdi); __ movq(rax, rcx); @@ -2285,7 +2282,7 @@ void Builtins::Generate_CallFunction(MacroAssembler* masm, __ movq(rcx, rax); __ Pop(rdi); __ Pop(rax); - __ SmiUntag(rax, rax); + __ SmiUntag(rax); } __ LoadTaggedPointerField( rdx, FieldOperand(rdi, JSFunction::kSharedFunctionInfoOffset)); @@ -2601,14 +2598,8 @@ void Builtins::Generate_Construct(MacroAssembler* masm) { } void Builtins::Generate_InterpreterOnStackReplacement(MacroAssembler* masm) { - // Lookup the function in the JavaScript frame. - __ movq(rax, Operand(rbp, StandardFrameConstants::kCallerFPOffset)); - __ movq(rax, Operand(rax, JavaScriptFrameConstants::kFunctionOffset)); - { FrameScope scope(masm, StackFrame::INTERNAL); - // Pass function as argument. - __ Push(rax); __ CallRuntime(Runtime::kCompileForOnStackReplacement); } @@ -2647,7 +2638,7 @@ void Builtins::Generate_WasmCompileLazy(MacroAssembler* masm) { // The function index was pushed to the stack by the caller as int32. __ Pop(r11); // Convert to Smi for the runtime call. - __ SmiTag(r11, r11); + __ SmiTag(r11); { HardAbortScope hard_abort(masm); // Avoid calls to Abort. FrameScope scope(masm, StackFrame::WASM_COMPILE_LAZY); @@ -2716,7 +2707,7 @@ void Builtins::Generate_CEntry(MacroAssembler* masm, int result_size, // If argv_mode == kArgvInRegister: // r15: pointer to the first argument -#ifdef _WIN64 +#ifdef V8_TARGET_OS_WIN // Windows 64-bit ABI passes arguments in rcx, rdx, r8, r9. It requires the // stack to be aligned to 16 bytes. It only allows a single-word to be // returned in register rax. Larger return sizes must be written to an address @@ -2738,7 +2729,7 @@ void Builtins::Generate_CEntry(MacroAssembler* masm, int result_size, const Register kCCallArg3 = rcx; const int kArgExtraStackSpace = 0; const int kMaxRegisterResultSize = 2; -#endif // _WIN64 +#endif // V8_TARGET_OS_WIN // Enter the exit frame that transitions from JavaScript to C++. int arg_stack_space = @@ -2809,7 +2800,7 @@ void Builtins::Generate_CEntry(MacroAssembler* masm, int result_size, IsolateAddressId::kPendingExceptionAddress, masm->isolate()); Operand pending_exception_operand = masm->ExternalReferenceAsOperand(pending_exception_address); - __ cmpq(r14, pending_exception_operand); + __ cmp_tagged(r14, pending_exception_operand); __ j(equal, &okay, Label::kNear); __ int3(); __ bind(&okay); |