summaryrefslogtreecommitdiff
path: root/deps/v8/src/compiler/arm64/code-generator-arm64.cc
diff options
context:
space:
mode:
Diffstat (limited to 'deps/v8/src/compiler/arm64/code-generator-arm64.cc')
-rw-r--r--deps/v8/src/compiler/arm64/code-generator-arm64.cc539
1 files changed, 411 insertions, 128 deletions
diff --git a/deps/v8/src/compiler/arm64/code-generator-arm64.cc b/deps/v8/src/compiler/arm64/code-generator-arm64.cc
index 456e7e7608..39c3314888 100644
--- a/deps/v8/src/compiler/arm64/code-generator-arm64.cc
+++ b/deps/v8/src/compiler/arm64/code-generator-arm64.cc
@@ -33,6 +33,24 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
return InputDoubleRegister(index);
}
+ CPURegister InputFloat32OrZeroRegister(size_t index) {
+ if (instr_->InputAt(index)->IsImmediate()) {
+ DCHECK(bit_cast<int32_t>(InputFloat32(index)) == 0);
+ return wzr;
+ }
+ DCHECK(instr_->InputAt(index)->IsFPRegister());
+ return InputDoubleRegister(index).S();
+ }
+
+ CPURegister InputFloat64OrZeroRegister(size_t index) {
+ if (instr_->InputAt(index)->IsImmediate()) {
+ DCHECK(bit_cast<int64_t>(InputDouble(index)) == 0);
+ return xzr;
+ }
+ DCHECK(instr_->InputAt(index)->IsDoubleRegister());
+ return InputDoubleRegister(index);
+ }
+
size_t OutputCount() { return instr_->OutputCount(); }
DoubleRegister OutputFloat32Register() { return OutputDoubleRegister().S(); }
@@ -101,6 +119,8 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
return Operand(InputRegister32(index), SXTB);
case kMode_Operand2_R_SXTH:
return Operand(InputRegister32(index), SXTH);
+ case kMode_Operand2_R_SXTW:
+ return Operand(InputRegister32(index), SXTW);
case kMode_MRI:
case kMode_MRR:
break;
@@ -129,6 +149,8 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
return Operand(InputRegister64(index), SXTB);
case kMode_Operand2_R_SXTH:
return Operand(InputRegister64(index), SXTH);
+ case kMode_Operand2_R_SXTW:
+ return Operand(InputRegister64(index), SXTW);
case kMode_MRI:
case kMode_MRR:
break;
@@ -141,7 +163,6 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
const size_t index = *first_index;
switch (AddressingModeField::decode(instr_->opcode())) {
case kMode_None:
- case kMode_Operand2_R_LSL_I:
case kMode_Operand2_R_LSR_I:
case kMode_Operand2_R_ASR_I:
case kMode_Operand2_R_ROR_I:
@@ -149,7 +170,12 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
case kMode_Operand2_R_UXTH:
case kMode_Operand2_R_SXTB:
case kMode_Operand2_R_SXTH:
+ case kMode_Operand2_R_SXTW:
break;
+ case kMode_Operand2_R_LSL_I:
+ *first_index += 3;
+ return MemOperand(InputRegister(index + 0), InputRegister(index + 1),
+ LSL, InputInt32(index + 2));
case kMode_MRI:
*first_index += 2;
return MemOperand(InputRegister(index + 0), InputInt32(index + 1));
@@ -183,9 +209,19 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
Constant constant = ToConstant(operand);
switch (constant.type()) {
case Constant::kInt32:
- return Operand(constant.ToInt32());
+ if (constant.rmode() == RelocInfo::WASM_MEMORY_SIZE_REFERENCE) {
+ return Operand(constant.ToInt32(), constant.rmode());
+ } else {
+ return Operand(constant.ToInt32());
+ }
case Constant::kInt64:
- return Operand(constant.ToInt64());
+ if (constant.rmode() == RelocInfo::WASM_MEMORY_REFERENCE ||
+ constant.rmode() == RelocInfo::WASM_GLOBAL_REFERENCE) {
+ return Operand(constant.ToInt64(), constant.rmode());
+ } else {
+ DCHECK(constant.rmode() != RelocInfo::WASM_MEMORY_SIZE_REFERENCE);
+ return Operand(constant.ToInt64());
+ }
case Constant::kFloat32:
return Operand(
isolate()->factory()->NewNumber(constant.ToFloat32(), TENURED));
@@ -206,7 +242,7 @@ class Arm64OperandConverter final : public InstructionOperandConverter {
MemOperand ToMemOperand(InstructionOperand* op, MacroAssembler* masm) const {
DCHECK_NOT_NULL(op);
- DCHECK(op->IsStackSlot() || op->IsDoubleStackSlot());
+ DCHECK(op->IsStackSlot() || op->IsFPStackSlot());
return SlotToMemOperand(AllocatedOperand::cast(op)->index(), masm);
}
@@ -272,7 +308,8 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
public:
OutOfLineRecordWrite(CodeGenerator* gen, Register object, Operand index,
Register value, Register scratch0, Register scratch1,
- RecordWriteMode mode)
+ RecordWriteMode mode,
+ UnwindingInfoWriter* unwinding_info_writer)
: OutOfLineCode(gen),
object_(object),
index_(index),
@@ -280,7 +317,8 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
scratch0_(scratch0),
scratch1_(scratch1),
mode_(mode),
- must_save_lr_(!gen->frame_access_state()->has_frame()) {}
+ must_save_lr_(!gen->frame_access_state()->has_frame()),
+ unwinding_info_writer_(unwinding_info_writer) {}
void Generate() final {
if (mode_ > RecordWriteMode::kValueIsPointer) {
@@ -297,6 +335,8 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
if (must_save_lr_) {
// We need to save and restore lr if the frame was elided.
__ Push(lr);
+ unwinding_info_writer_->MarkLinkRegisterOnTopOfStack(__ pc_offset(),
+ __ StackPointer());
}
RecordWriteStub stub(isolate(), object_, scratch0_, scratch1_,
remembered_set_action, save_fp_mode);
@@ -304,6 +344,7 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
__ CallStub(&stub);
if (must_save_lr_) {
__ Pop(lr);
+ unwinding_info_writer_->MarkPopLinkRegisterFromTopOfStack(__ pc_offset());
}
}
@@ -315,6 +356,7 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
Register const scratch1_;
RecordWriteMode const mode_;
bool must_save_lr_;
+ UnwindingInfoWriter* const unwinding_info_writer_;
};
@@ -363,6 +405,10 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
case kUnorderedEqual:
case kUnorderedNotEqual:
break;
+ case kPositiveOrZero:
+ return pl;
+ case kNegative:
+ return mi;
}
UNREACHABLE();
return nv;
@@ -370,6 +416,17 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
} // namespace
+#define ASSEMBLE_BOUNDS_CHECK(offset, length, out_of_bounds) \
+ do { \
+ if (length.IsImmediate() && \
+ base::bits::IsPowerOfTwo64(length.ImmediateValue())) { \
+ __ Tst(offset, ~(length.ImmediateValue() - 1)); \
+ __ B(ne, out_of_bounds); \
+ } else { \
+ __ Cmp(offset, length); \
+ __ B(hs, out_of_bounds); \
+ } \
+ } while (0)
#define ASSEMBLE_CHECKED_LOAD_FLOAT(width) \
do { \
@@ -377,84 +434,72 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
auto buffer = i.InputRegister(0); \
auto offset = i.InputRegister32(1); \
auto length = i.InputOperand32(2); \
- __ Cmp(offset, length); \
auto ool = new (zone()) OutOfLineLoadNaN##width(this, result); \
- __ B(hs, ool->entry()); \
+ ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \
__ Ldr(result, MemOperand(buffer, offset, UXTW)); \
__ Bind(ool->exit()); \
} while (0)
-
#define ASSEMBLE_CHECKED_LOAD_INTEGER(asm_instr) \
do { \
auto result = i.OutputRegister32(); \
auto buffer = i.InputRegister(0); \
auto offset = i.InputRegister32(1); \
auto length = i.InputOperand32(2); \
- __ Cmp(offset, length); \
auto ool = new (zone()) OutOfLineLoadZero(this, result); \
- __ B(hs, ool->entry()); \
+ ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \
__ asm_instr(result, MemOperand(buffer, offset, UXTW)); \
__ Bind(ool->exit()); \
} while (0)
-
#define ASSEMBLE_CHECKED_LOAD_INTEGER_64(asm_instr) \
do { \
auto result = i.OutputRegister(); \
auto buffer = i.InputRegister(0); \
auto offset = i.InputRegister32(1); \
auto length = i.InputOperand32(2); \
- __ Cmp(offset, length); \
auto ool = new (zone()) OutOfLineLoadZero(this, result); \
- __ B(hs, ool->entry()); \
+ ASSEMBLE_BOUNDS_CHECK(offset, length, ool->entry()); \
__ asm_instr(result, MemOperand(buffer, offset, UXTW)); \
__ Bind(ool->exit()); \
} while (0)
-
-#define ASSEMBLE_CHECKED_STORE_FLOAT(width) \
- do { \
- auto buffer = i.InputRegister(0); \
- auto offset = i.InputRegister32(1); \
- auto length = i.InputOperand32(2); \
- auto value = i.InputFloat##width##Register(3); \
- __ Cmp(offset, length); \
- Label done; \
- __ B(hs, &done); \
- __ Str(value, MemOperand(buffer, offset, UXTW)); \
- __ Bind(&done); \
+#define ASSEMBLE_CHECKED_STORE_FLOAT(width) \
+ do { \
+ auto buffer = i.InputRegister(0); \
+ auto offset = i.InputRegister32(1); \
+ auto length = i.InputOperand32(2); \
+ auto value = i.InputFloat##width##OrZeroRegister(3); \
+ Label done; \
+ ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \
+ __ Str(value, MemOperand(buffer, offset, UXTW)); \
+ __ Bind(&done); \
} while (0)
-
#define ASSEMBLE_CHECKED_STORE_INTEGER(asm_instr) \
do { \
auto buffer = i.InputRegister(0); \
auto offset = i.InputRegister32(1); \
auto length = i.InputOperand32(2); \
- auto value = i.InputRegister32(3); \
- __ Cmp(offset, length); \
+ auto value = i.InputOrZeroRegister32(3); \
Label done; \
- __ B(hs, &done); \
+ ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \
__ asm_instr(value, MemOperand(buffer, offset, UXTW)); \
__ Bind(&done); \
} while (0)
-
#define ASSEMBLE_CHECKED_STORE_INTEGER_64(asm_instr) \
do { \
auto buffer = i.InputRegister(0); \
auto offset = i.InputRegister32(1); \
auto length = i.InputOperand32(2); \
- auto value = i.InputRegister(3); \
- __ Cmp(offset, length); \
+ auto value = i.InputOrZeroRegister64(3); \
Label done; \
- __ B(hs, &done); \
+ ASSEMBLE_BOUNDS_CHECK(offset, length, &done); \
__ asm_instr(value, MemOperand(buffer, offset, UXTW)); \
__ Bind(&done); \
} while (0)
-
#define ASSEMBLE_SHIFT(asm_instr, width) \
do { \
if (instr->InputAt(1)->IsRegister()) { \
@@ -468,6 +513,35 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
} \
} while (0)
+#define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \
+ do { \
+ __ asm_instr(i.OutputRegister(), \
+ MemOperand(i.InputRegister(0), i.InputRegister(1))); \
+ __ Dmb(InnerShareable, BarrierAll); \
+ } while (0)
+
+#define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \
+ do { \
+ __ Dmb(InnerShareable, BarrierAll); \
+ __ asm_instr(i.InputRegister(2), \
+ MemOperand(i.InputRegister(0), i.InputRegister(1))); \
+ __ Dmb(InnerShareable, BarrierAll); \
+ } while (0)
+
+#define ASSEMBLE_IEEE754_BINOP(name) \
+ do { \
+ FrameScope scope(masm(), StackFrame::MANUAL); \
+ __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
+ 0, 2); \
+ } while (0)
+
+#define ASSEMBLE_IEEE754_UNOP(name) \
+ do { \
+ FrameScope scope(masm(), StackFrame::MANUAL); \
+ __ CallCFunction(ExternalReference::ieee754_##name##_function(isolate()), \
+ 0, 1); \
+ } while (0)
+
void CodeGenerator::AssembleDeconstructFrame() {
const CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
if (descriptor->IsCFunctionCall() || descriptor->UseNativeStack()) {
@@ -476,23 +550,11 @@ void CodeGenerator::AssembleDeconstructFrame() {
__ Mov(jssp, fp);
}
__ Pop(fp, lr);
-}
-void CodeGenerator::AssembleDeconstructActivationRecord(int stack_param_delta) {
- int sp_slot_delta = TailCallFrameStackSlotDelta(stack_param_delta);
- if (sp_slot_delta > 0) {
- __ Drop(sp_slot_delta);
- }
- frame_access_state()->SetFrameAccessToDefault();
+ unwinding_info_writer_.MarkFrameDeconstructed(__ pc_offset());
}
-
-void CodeGenerator::AssemblePrepareTailCall(int stack_param_delta) {
- int sp_slot_delta = TailCallFrameStackSlotDelta(stack_param_delta);
- if (sp_slot_delta < 0) {
- __ Claim(-sp_slot_delta);
- frame_access_state()->IncreaseSPDelta(-sp_slot_delta);
- }
+void CodeGenerator::AssemblePrepareTailCall() {
if (frame_access_state()->has_frame()) {
__ Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
__ Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
@@ -525,8 +587,41 @@ void CodeGenerator::AssemblePopArgumentsAdaptorFrame(Register args_reg,
__ bind(&done);
}
+namespace {
+
+void AdjustStackPointerForTailCall(MacroAssembler* masm,
+ FrameAccessState* state,
+ int new_slot_above_sp,
+ bool allow_shrinkage = true) {
+ int current_sp_offset = state->GetSPToFPSlotCount() +
+ StandardFrameConstants::kFixedSlotCountAboveFp;
+ int stack_slot_delta = new_slot_above_sp - current_sp_offset;
+ if (stack_slot_delta > 0) {
+ masm->Claim(stack_slot_delta);
+ state->IncreaseSPDelta(stack_slot_delta);
+ } else if (allow_shrinkage && stack_slot_delta < 0) {
+ masm->Drop(-stack_slot_delta);
+ state->IncreaseSPDelta(stack_slot_delta);
+ }
+}
+
+} // namespace
+
+void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
+ int first_unused_stack_slot) {
+ AdjustStackPointerForTailCall(masm(), frame_access_state(),
+ first_unused_stack_slot, false);
+}
+
+void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
+ int first_unused_stack_slot) {
+ AdjustStackPointerForTailCall(masm(), frame_access_state(),
+ first_unused_stack_slot);
+}
+
// Assembles an instruction after register allocation, producing machine code.
-void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
+CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
+ Instruction* instr) {
Arm64OperandConverter i(this, instr);
InstructionCode opcode = instr->opcode();
ArchOpcode arch_opcode = ArchOpcodeField::decode(opcode);
@@ -559,8 +654,6 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
}
case kArchTailCallCodeObjectFromJSFunction:
case kArchTailCallCodeObject: {
- int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
- AssembleDeconstructActivationRecord(stack_param_delta);
if (arch_opcode == kArchTailCallCodeObjectFromJSFunction) {
AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
i.TempRegister(0), i.TempRegister(1),
@@ -574,7 +667,17 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Add(target, target, Code::kHeaderSize - kHeapObjectTag);
__ Jump(target);
}
+ unwinding_info_writer_.MarkBlockWillExit();
frame_access_state()->ClearSPDelta();
+ frame_access_state()->SetFrameAccessToDefault();
+ break;
+ }
+ case kArchTailCallAddress: {
+ CHECK(!instr->InputAt(0)->IsImmediate());
+ __ Jump(i.InputRegister(0));
+ unwinding_info_writer_.MarkBlockWillExit();
+ frame_access_state()->ClearSPDelta();
+ frame_access_state()->SetFrameAccessToDefault();
break;
}
case kArchCallJSFunction: {
@@ -617,8 +720,6 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ cmp(cp, temp);
__ Assert(eq, kWrongFunctionContext);
}
- int stack_param_delta = i.InputInt32(instr->InputCount() - 1);
- AssembleDeconstructActivationRecord(stack_param_delta);
if (arch_opcode == kArchTailCallJSFunctionFromJSFunction) {
AssemblePopArgumentsAdaptorFrame(kJavaScriptCallArgCountRegister,
i.TempRegister(0), i.TempRegister(1),
@@ -627,6 +728,7 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
__ Jump(x10);
frame_access_state()->ClearSPDelta();
+ frame_access_state()->SetFrameAccessToDefault();
break;
}
case kArchPrepareCallCFunction:
@@ -636,7 +738,7 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
UNREACHABLE();
break;
case kArchPrepareTailCall:
- AssemblePrepareTailCall(i.InputInt32(instr->InputCount() - 1));
+ AssemblePrepareTailCall();
break;
case kArchCallCFunction: {
int const num_parameters = MiscField::decode(instr->opcode());
@@ -661,6 +763,17 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
case kArchLookupSwitch:
AssembleArchLookupSwitch(instr);
break;
+ case kArchDebugBreak:
+ __ Debug("kArchDebugBreak", 0, BREAK);
+ break;
+ case kArchImpossible:
+ __ Abort(kConversionFromImpossibleValue);
+ break;
+ case kArchComment: {
+ Address comment_string = i.InputExternalReference(0).address();
+ __ RecordComment(reinterpret_cast<const char*>(comment_string));
+ break;
+ }
case kArchNop:
case kArchThrowTerminator:
// don't emit code for nops.
@@ -670,7 +783,9 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
BuildTranslation(instr, -1, 0, OutputFrameStateCombine::Ignore());
Deoptimizer::BailoutType bailout_type =
Deoptimizer::BailoutType(MiscField::decode(instr->opcode()));
- AssembleDeoptimizerCall(deopt_state_id, bailout_type);
+ CodeGenResult result =
+ AssembleDeoptimizerCall(deopt_state_id, bailout_type);
+ if (result != kSuccess) return result;
break;
}
case kArchRet:
@@ -708,8 +823,9 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
Register value = i.InputRegister(2);
Register scratch0 = i.TempRegister(0);
Register scratch1 = i.TempRegister(1);
- auto ool = new (zone()) OutOfLineRecordWrite(this, object, index, value,
- scratch0, scratch1, mode);
+ auto ool = new (zone())
+ OutOfLineRecordWrite(this, object, index, value, scratch0, scratch1,
+ mode, &unwinding_info_writer_);
__ Str(value, MemOperand(object, index));
__ CheckPageFlagSet(object, scratch0,
MemoryChunk::kPointersFromHereAreInterestingMask,
@@ -729,6 +845,71 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Add(i.OutputRegister(0), base, Operand(offset.offset()));
break;
}
+ case kIeee754Float64Acos:
+ ASSEMBLE_IEEE754_UNOP(acos);
+ break;
+ case kIeee754Float64Acosh:
+ ASSEMBLE_IEEE754_UNOP(acosh);
+ break;
+ case kIeee754Float64Asin:
+ ASSEMBLE_IEEE754_UNOP(asin);
+ break;
+ case kIeee754Float64Asinh:
+ ASSEMBLE_IEEE754_UNOP(asinh);
+ break;
+ case kIeee754Float64Atan:
+ ASSEMBLE_IEEE754_UNOP(atan);
+ break;
+ case kIeee754Float64Atanh:
+ ASSEMBLE_IEEE754_UNOP(atanh);
+ break;
+ case kIeee754Float64Atan2:
+ ASSEMBLE_IEEE754_BINOP(atan2);
+ break;
+ case kIeee754Float64Cos:
+ ASSEMBLE_IEEE754_UNOP(cos);
+ break;
+ case kIeee754Float64Cosh:
+ ASSEMBLE_IEEE754_UNOP(cosh);
+ break;
+ case kIeee754Float64Cbrt:
+ ASSEMBLE_IEEE754_UNOP(cbrt);
+ break;
+ case kIeee754Float64Exp:
+ ASSEMBLE_IEEE754_UNOP(exp);
+ break;
+ case kIeee754Float64Expm1:
+ ASSEMBLE_IEEE754_UNOP(expm1);
+ break;
+ case kIeee754Float64Log:
+ ASSEMBLE_IEEE754_UNOP(log);
+ break;
+ case kIeee754Float64Log1p:
+ ASSEMBLE_IEEE754_UNOP(log1p);
+ break;
+ case kIeee754Float64Log2:
+ ASSEMBLE_IEEE754_UNOP(log2);
+ break;
+ case kIeee754Float64Log10:
+ ASSEMBLE_IEEE754_UNOP(log10);
+ break;
+ case kIeee754Float64Pow: {
+ MathPowStub stub(isolate(), MathPowStub::DOUBLE);
+ __ CallStub(&stub);
+ break;
+ }
+ case kIeee754Float64Sin:
+ ASSEMBLE_IEEE754_UNOP(sin);
+ break;
+ case kIeee754Float64Sinh:
+ ASSEMBLE_IEEE754_UNOP(sinh);
+ break;
+ case kIeee754Float64Tan:
+ ASSEMBLE_IEEE754_UNOP(tan);
+ break;
+ case kIeee754Float64Tanh:
+ ASSEMBLE_IEEE754_UNOP(tanh);
+ break;
case kArm64Float32RoundDown:
__ Frintm(i.OutputFloat32Register(), i.InputFloat32Register(0));
break;
@@ -775,12 +956,34 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
}
break;
case kArm64And:
- __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
- i.InputOperand2_64(1));
+ if (FlagsModeField::decode(opcode) != kFlags_none) {
+ // The ands instruction only sets N and Z, so only the following
+ // conditions make sense.
+ DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
+ FlagsConditionField::decode(opcode) == kNotEqual ||
+ FlagsConditionField::decode(opcode) == kPositiveOrZero ||
+ FlagsConditionField::decode(opcode) == kNegative);
+ __ Ands(i.OutputRegister(), i.InputOrZeroRegister64(0),
+ i.InputOperand2_64(1));
+ } else {
+ __ And(i.OutputRegister(), i.InputOrZeroRegister64(0),
+ i.InputOperand2_64(1));
+ }
break;
case kArm64And32:
- __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
- i.InputOperand2_32(1));
+ if (FlagsModeField::decode(opcode) != kFlags_none) {
+ // The ands instruction only sets N and Z, so only the following
+ // conditions make sense.
+ DCHECK(FlagsConditionField::decode(opcode) == kEqual ||
+ FlagsConditionField::decode(opcode) == kNotEqual ||
+ FlagsConditionField::decode(opcode) == kPositiveOrZero ||
+ FlagsConditionField::decode(opcode) == kNegative);
+ __ Ands(i.OutputRegister32(), i.InputOrZeroRegister32(0),
+ i.InputOperand2_32(1));
+ } else {
+ __ And(i.OutputRegister32(), i.InputOrZeroRegister32(0),
+ i.InputOperand2_32(1));
+ }
break;
case kArm64Bic:
__ Bic(i.OutputRegister(), i.InputOrZeroRegister64(0),
@@ -983,6 +1186,7 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
// Pseudo instructions turned into tbz/tbnz in AssembleArchBranch.
break;
case kArm64CompareAndBranch32:
+ case kArm64CompareAndBranch:
// Pseudo instruction turned into cbz/cbnz in AssembleArchBranch.
break;
case kArm64ClaimCSP: {
@@ -1038,7 +1242,7 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
Register prev = __ StackPointer();
__ SetStackPointer(arch_opcode == kArm64PokeCSP ? csp : jssp);
Operand operand(i.InputInt32(1) * kPointerSize);
- if (instr->InputAt(0)->IsDoubleRegister()) {
+ if (instr->InputAt(0)->IsFPRegister()) {
__ Poke(i.InputFloat64Register(0), operand);
} else {
__ Poke(i.InputRegister(0), operand);
@@ -1048,7 +1252,7 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
}
case kArm64PokePair: {
int slot = i.InputInt32(2) - 1;
- if (instr->InputAt(0)->IsDoubleRegister()) {
+ if (instr->InputAt(0)->IsFPRegister()) {
__ PokePair(i.InputFloat64Register(1), i.InputFloat64Register(0),
slot * kPointerSize);
} else {
@@ -1070,25 +1274,25 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Rbit(i.OutputRegister32(), i.InputRegister32(0));
break;
case kArm64Cmp:
- __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand(1));
+ __ Cmp(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
break;
case kArm64Cmp32:
__ Cmp(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
break;
case kArm64Cmn:
- __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand(1));
+ __ Cmn(i.InputOrZeroRegister64(0), i.InputOperand2_64(1));
break;
case kArm64Cmn32:
__ Cmn(i.InputOrZeroRegister32(0), i.InputOperand2_32(1));
break;
case kArm64Tst:
- __ Tst(i.InputRegister(0), i.InputOperand(1));
+ __ Tst(i.InputOrZeroRegister64(0), i.InputOperand(1));
break;
case kArm64Tst32:
- __ Tst(i.InputRegister32(0), i.InputOperand32(1));
+ __ Tst(i.InputOrZeroRegister32(0), i.InputOperand32(1));
break;
case kArm64Float32Cmp:
- if (instr->InputAt(1)->IsDoubleRegister()) {
+ if (instr->InputAt(1)->IsFPRegister()) {
__ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
} else {
DCHECK(instr->InputAt(1)->IsImmediate());
@@ -1113,26 +1317,17 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Fdiv(i.OutputFloat32Register(), i.InputFloat32Register(0),
i.InputFloat32Register(1));
break;
- case kArm64Float32Max:
- // (b < a) ? a : b
- __ Fcmp(i.InputFloat32Register(1), i.InputFloat32Register(0));
- __ Fcsel(i.OutputFloat32Register(), i.InputFloat32Register(0),
- i.InputFloat32Register(1), lo);
- break;
- case kArm64Float32Min:
- // (a < b) ? a : b
- __ Fcmp(i.InputFloat32Register(0), i.InputFloat32Register(1));
- __ Fcsel(i.OutputFloat32Register(), i.InputFloat32Register(0),
- i.InputFloat32Register(1), lo);
- break;
case kArm64Float32Abs:
__ Fabs(i.OutputFloat32Register(), i.InputFloat32Register(0));
break;
+ case kArm64Float32Neg:
+ __ Fneg(i.OutputFloat32Register(), i.InputFloat32Register(0));
+ break;
case kArm64Float32Sqrt:
__ Fsqrt(i.OutputFloat32Register(), i.InputFloat32Register(0));
break;
case kArm64Float64Cmp:
- if (instr->InputAt(1)->IsDoubleRegister()) {
+ if (instr->InputAt(1)->IsFPRegister()) {
__ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
} else {
DCHECK(instr->InputAt(1)->IsImmediate());
@@ -1168,18 +1363,26 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
0, 2);
break;
}
- case kArm64Float64Max:
- // (b < a) ? a : b
- __ Fcmp(i.InputDoubleRegister(1), i.InputDoubleRegister(0));
- __ Fcsel(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
- i.InputDoubleRegister(1), lo);
- break;
- case kArm64Float64Min:
- // (a < b) ? a : b
- __ Fcmp(i.InputDoubleRegister(0), i.InputDoubleRegister(1));
- __ Fcsel(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
- i.InputDoubleRegister(1), lo);
+ case kArm64Float32Max: {
+ __ Fmax(i.OutputFloat32Register(), i.InputFloat32Register(0),
+ i.InputFloat32Register(1));
+ break;
+ }
+ case kArm64Float64Max: {
+ __ Fmax(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
+ i.InputDoubleRegister(1));
+ break;
+ }
+ case kArm64Float32Min: {
+ __ Fmin(i.OutputFloat32Register(), i.InputFloat32Register(0),
+ i.InputFloat32Register(1));
+ break;
+ }
+ case kArm64Float64Min: {
+ __ Fmin(i.OutputDoubleRegister(), i.InputDoubleRegister(0),
+ i.InputDoubleRegister(1));
break;
+ }
case kArm64Float64Abs:
__ Fabs(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
break;
@@ -1197,12 +1400,21 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
break;
case kArm64Float32ToInt32:
__ Fcvtzs(i.OutputRegister32(), i.InputFloat32Register(0));
+ // Avoid INT32_MAX as an overflow indicator and use INT32_MIN instead,
+ // because INT32_MIN allows easier out-of-bounds detection.
+ __ Cmn(i.OutputRegister32(), 1);
+ __ Csinc(i.OutputRegister32(), i.OutputRegister32(), i.OutputRegister32(),
+ vc);
break;
case kArm64Float64ToInt32:
__ Fcvtzs(i.OutputRegister32(), i.InputDoubleRegister(0));
break;
case kArm64Float32ToUint32:
__ Fcvtzu(i.OutputRegister32(), i.InputFloat32Register(0));
+ // Avoid UINT32_MAX as an overflow indicator and use 0 instead,
+ // because 0 allows easier out-of-bounds detection.
+ __ Cmn(i.OutputRegister32(), 1);
+ __ Adc(i.OutputRegister32(), i.OutputRegister32(), Operand(0));
break;
case kArm64Float64ToUint32:
__ Fcvtzu(i.OutputRegister32(), i.InputDoubleRegister(0));
@@ -1305,6 +1517,9 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
case kArm64Float64MoveU64:
__ Fmov(i.OutputFloat64Register(), i.InputRegister(0));
break;
+ case kArm64Float64SilenceNaN:
+ __ CanonicalizeNaN(i.OutputDoubleRegister(), i.InputDoubleRegister(0));
+ break;
case kArm64U64MoveFloat64:
__ Fmov(i.OutputRegister(), i.InputDoubleRegister(0));
break;
@@ -1315,7 +1530,7 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Ldrsb(i.OutputRegister(), i.MemoryOperand());
break;
case kArm64Strb:
- __ Strb(i.InputRegister(2), i.MemoryOperand());
+ __ Strb(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
break;
case kArm64Ldrh:
__ Ldrh(i.OutputRegister(), i.MemoryOperand());
@@ -1324,31 +1539,34 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
__ Ldrsh(i.OutputRegister(), i.MemoryOperand());
break;
case kArm64Strh:
- __ Strh(i.InputRegister(2), i.MemoryOperand());
+ __ Strh(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
+ break;
+ case kArm64Ldrsw:
+ __ Ldrsw(i.OutputRegister(), i.MemoryOperand());
break;
case kArm64LdrW:
__ Ldr(i.OutputRegister32(), i.MemoryOperand());
break;
case kArm64StrW:
- __ Str(i.InputRegister32(2), i.MemoryOperand());
+ __ Str(i.InputOrZeroRegister32(0), i.MemoryOperand(1));
break;
case kArm64Ldr:
__ Ldr(i.OutputRegister(), i.MemoryOperand());
break;
case kArm64Str:
- __ Str(i.InputRegister(2), i.MemoryOperand());
+ __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
break;
case kArm64LdrS:
__ Ldr(i.OutputDoubleRegister().S(), i.MemoryOperand());
break;
case kArm64StrS:
- __ Str(i.InputDoubleRegister(2).S(), i.MemoryOperand());
+ __ Str(i.InputFloat32OrZeroRegister(0), i.MemoryOperand(1));
break;
case kArm64LdrD:
__ Ldr(i.OutputDoubleRegister(), i.MemoryOperand());
break;
case kArm64StrD:
- __ Str(i.InputDoubleRegister(2), i.MemoryOperand());
+ __ Str(i.InputFloat64OrZeroRegister(0), i.MemoryOperand(1));
break;
case kCheckedLoadInt8:
ASSEMBLE_CHECKED_LOAD_INTEGER(Ldrsb);
@@ -1392,7 +1610,37 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
case kCheckedStoreFloat64:
ASSEMBLE_CHECKED_STORE_FLOAT(64);
break;
+ case kAtomicLoadInt8:
+ ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb);
+ break;
+ case kAtomicLoadUint8:
+ ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb);
+ break;
+ case kAtomicLoadInt16:
+ ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh);
+ break;
+ case kAtomicLoadUint16:
+ ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh);
+ break;
+ case kAtomicLoadWord32:
+ __ Ldr(i.OutputRegister32(),
+ MemOperand(i.InputRegister(0), i.InputRegister(1)));
+ __ Dmb(InnerShareable, BarrierAll);
+ break;
+ case kAtomicStoreWord8:
+ ASSEMBLE_ATOMIC_STORE_INTEGER(Strb);
+ break;
+ case kAtomicStoreWord16:
+ ASSEMBLE_ATOMIC_STORE_INTEGER(Strh);
+ break;
+ case kAtomicStoreWord32:
+ __ Dmb(InnerShareable, BarrierAll);
+ __ Str(i.InputRegister32(2),
+ MemOperand(i.InputRegister(0), i.InputRegister(1)));
+ __ Dmb(InnerShareable, BarrierAll);
+ break;
}
+ return kSuccess;
} // NOLINT(readability/fn_size)
@@ -1415,6 +1663,17 @@ void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) {
default:
UNREACHABLE();
}
+ } else if (opcode == kArm64CompareAndBranch) {
+ switch (condition) {
+ case kEqual:
+ __ Cbz(i.InputRegister64(0), tlabel);
+ break;
+ case kNotEqual:
+ __ Cbnz(i.InputRegister64(0), tlabel);
+ break;
+ default:
+ UNREACHABLE();
+ }
} else if (opcode == kArm64TestAndBranch32) {
switch (condition) {
case kEqual:
@@ -1495,30 +1754,52 @@ void CodeGenerator::AssembleArchTableSwitch(Instruction* instr) {
__ EndBlockPools();
}
-
-void CodeGenerator::AssembleDeoptimizerCall(
+CodeGenerator::CodeGenResult CodeGenerator::AssembleDeoptimizerCall(
int deoptimization_id, Deoptimizer::BailoutType bailout_type) {
Address deopt_entry = Deoptimizer::GetDeoptimizationEntry(
isolate(), deoptimization_id, bailout_type);
+ if (deopt_entry == nullptr) return kTooManyDeoptimizationBailouts;
+ DeoptimizeReason deoptimization_reason =
+ GetDeoptimizationReason(deoptimization_id);
+ __ RecordDeoptReason(deoptimization_reason, 0, deoptimization_id);
__ Call(deopt_entry, RelocInfo::RUNTIME_ENTRY);
+ return kSuccess;
}
-void CodeGenerator::AssembleSetupStackPointer() {
- const CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
+void CodeGenerator::FinishFrame(Frame* frame) {
+ frame->AlignFrame(16);
+ CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
+
if (descriptor->UseNativeStack() || descriptor->IsCFunctionCall()) {
__ SetStackPointer(csp);
} else {
__ SetStackPointer(jssp);
}
+
+ // Save FP registers.
+ CPURegList saves_fp = CPURegList(CPURegister::kFPRegister, kDRegSizeInBits,
+ descriptor->CalleeSavedFPRegisters());
+ int saved_count = saves_fp.Count();
+ if (saved_count != 0) {
+ DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list());
+ frame->AllocateSavedCalleeRegisterSlots(saved_count *
+ (kDoubleSize / kPointerSize));
+ }
+
+ CPURegList saves = CPURegList(CPURegister::kRegister, kXRegSizeInBits,
+ descriptor->CalleeSavedRegisters());
+ saved_count = saves.Count();
+ if (saved_count != 0) {
+ frame->AllocateSavedCalleeRegisterSlots(saved_count);
+ }
}
-void CodeGenerator::AssemblePrologue() {
+void CodeGenerator::AssembleConstructFrame() {
CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
if (descriptor->UseNativeStack()) {
__ AssertCspAligned();
}
- int stack_shrink_slots = frame()->GetSpillSlotCount();
if (frame_access_state()->has_frame()) {
if (descriptor->IsJSFunctionCall()) {
DCHECK(!descriptor->UseNativeStack());
@@ -1527,14 +1808,20 @@ void CodeGenerator::AssemblePrologue() {
if (descriptor->IsCFunctionCall()) {
__ Push(lr, fp);
__ Mov(fp, masm_.StackPointer());
- __ Claim(stack_shrink_slots);
+ __ Claim(frame()->GetSpillSlotCount());
} else {
__ StubPrologue(info()->GetOutputStackFrameType(),
frame()->GetTotalFrameSlotCount());
}
}
+
+ if (!info()->GeneratePreagedPrologue()) {
+ unwinding_info_writer_.MarkFrameConstructed(__ pc_offset());
+ }
}
+ int shrink_slots = frame()->GetSpillSlotCount();
+
if (info()->is_osr()) {
// TurboFan OSR-compiled functions cannot be entered directly.
__ Abort(kShouldNotDirectlyEnterOsrFunction);
@@ -1545,11 +1832,11 @@ void CodeGenerator::AssemblePrologue() {
// remaining stack slots.
if (FLAG_code_comments) __ RecordComment("-- OSR entrypoint --");
osr_pc_offset_ = __ pc_offset();
- stack_shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots();
+ shrink_slots -= OsrHelper(info()).UnoptimizedFrameSlots();
}
if (descriptor->IsJSFunctionCall()) {
- __ Claim(stack_shrink_slots);
+ __ Claim(shrink_slots);
}
// Save FP registers.
@@ -1559,8 +1846,6 @@ void CodeGenerator::AssemblePrologue() {
if (saved_count != 0) {
DCHECK(saves_fp.list() == CPURegList::GetCalleeSavedFP().list());
__ PushCPURegList(saves_fp);
- frame()->AllocateSavedCalleeRegisterSlots(saved_count *
- (kDoubleSize / kPointerSize));
}
// Save registers.
// TODO(palfia): TF save list is not in sync with
@@ -1571,7 +1856,6 @@ void CodeGenerator::AssemblePrologue() {
saved_count = saves.Count();
if (saved_count != 0) {
__ PushCPURegList(saves);
- frame()->AllocateSavedCalleeRegisterSlots(saved_count);
}
}
@@ -1593,6 +1877,8 @@ void CodeGenerator::AssembleReturn() {
__ PopCPURegList(saves_fp);
}
+ unwinding_info_writer_.MarkBlockWillExit();
+
int pop_count = static_cast<int>(descriptor->StackParameterCount());
if (descriptor->IsCFunctionCall()) {
AssembleDeconstructFrame();
@@ -1668,11 +1954,11 @@ void CodeGenerator::AssembleMove(InstructionOperand* source,
__ Str(dst, g.ToMemOperand(destination, masm()));
}
} else if (src.type() == Constant::kFloat32) {
- if (destination->IsDoubleRegister()) {
+ if (destination->IsFPRegister()) {
FPRegister dst = g.ToDoubleRegister(destination).S();
__ Fmov(dst, src.ToFloat32());
} else {
- DCHECK(destination->IsDoubleStackSlot());
+ DCHECK(destination->IsFPStackSlot());
UseScratchRegisterScope scope(masm());
FPRegister temp = scope.AcquireS();
__ Fmov(temp, src.ToFloat32());
@@ -1680,30 +1966,30 @@ void CodeGenerator::AssembleMove(InstructionOperand* source,
}
} else {
DCHECK_EQ(Constant::kFloat64, src.type());
- if (destination->IsDoubleRegister()) {
+ if (destination->IsFPRegister()) {
FPRegister dst = g.ToDoubleRegister(destination);
__ Fmov(dst, src.ToFloat64());
} else {
- DCHECK(destination->IsDoubleStackSlot());
+ DCHECK(destination->IsFPStackSlot());
UseScratchRegisterScope scope(masm());
FPRegister temp = scope.AcquireD();
__ Fmov(temp, src.ToFloat64());
__ Str(temp, g.ToMemOperand(destination, masm()));
}
}
- } else if (source->IsDoubleRegister()) {
+ } else if (source->IsFPRegister()) {
FPRegister src = g.ToDoubleRegister(source);
- if (destination->IsDoubleRegister()) {
+ if (destination->IsFPRegister()) {
FPRegister dst = g.ToDoubleRegister(destination);
__ Fmov(dst, src);
} else {
- DCHECK(destination->IsDoubleStackSlot());
+ DCHECK(destination->IsFPStackSlot());
__ Str(src, g.ToMemOperand(destination, masm()));
}
- } else if (source->IsDoubleStackSlot()) {
- DCHECK(destination->IsDoubleRegister() || destination->IsDoubleStackSlot());
+ } else if (source->IsFPStackSlot()) {
+ DCHECK(destination->IsFPRegister() || destination->IsFPStackSlot());
MemOperand src = g.ToMemOperand(source, masm());
- if (destination->IsDoubleRegister()) {
+ if (destination->IsFPRegister()) {
__ Ldr(g.ToDoubleRegister(destination), src);
} else {
UseScratchRegisterScope scope(masm());
@@ -1739,7 +2025,7 @@ void CodeGenerator::AssembleSwap(InstructionOperand* source,
__ Ldr(src, dst);
__ Str(temp, dst);
}
- } else if (source->IsStackSlot() || source->IsDoubleStackSlot()) {
+ } else if (source->IsStackSlot() || source->IsFPStackSlot()) {
UseScratchRegisterScope scope(masm());
DoubleRegister temp_0 = scope.AcquireD();
DoubleRegister temp_1 = scope.AcquireD();
@@ -1749,17 +2035,17 @@ void CodeGenerator::AssembleSwap(InstructionOperand* source,
__ Ldr(temp_1, dst);
__ Str(temp_0, dst);
__ Str(temp_1, src);
- } else if (source->IsDoubleRegister()) {
+ } else if (source->IsFPRegister()) {
UseScratchRegisterScope scope(masm());
FPRegister temp = scope.AcquireD();
FPRegister src = g.ToDoubleRegister(source);
- if (destination->IsDoubleRegister()) {
+ if (destination->IsFPRegister()) {
FPRegister dst = g.ToDoubleRegister(destination);
__ Fmov(temp, src);
__ Fmov(src, dst);
__ Fmov(dst, temp);
} else {
- DCHECK(destination->IsDoubleStackSlot());
+ DCHECK(destination->IsFPStackSlot());
MemOperand dst = g.ToMemOperand(destination, masm());
__ Fmov(temp, src);
__ Ldr(src, dst);
@@ -1778,9 +2064,6 @@ void CodeGenerator::AssembleJumpTable(Label** targets, size_t target_count) {
}
-void CodeGenerator::AddNopForSmiCodeInlining() { __ movz(xzr, 0); }
-
-
void CodeGenerator::EnsureSpaceForLazyDeopt() {
if (!info()->ShouldEnsureSpaceForLazyDeopt()) {
return;
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-02 18:19:57 +0000