diff options
Diffstat (limited to 'deps/v8/src/compiler/wasm-compiler.cc')
| -rw-r--r-- | deps/v8/src/compiler/wasm-compiler.cc | 982 |
1 files changed, 686 insertions, 296 deletions
diff --git a/deps/v8/src/compiler/wasm-compiler.cc b/deps/v8/src/compiler/wasm-compiler.cc index bc731b2bb8..a04c7b3e5d 100644 --- a/deps/v8/src/compiler/wasm-compiler.cc +++ b/deps/v8/src/compiler/wasm-compiler.cc @@ -13,6 +13,7 @@ #include "src/builtins/builtins.h" #include "src/code-factory.h" #include "src/compiler/access-builder.h" +#include "src/compiler/code-generator.h" #include "src/compiler/common-operator.h" #include "src/compiler/compiler-source-position-table.h" #include "src/compiler/diamond.h" @@ -31,6 +32,7 @@ #include "src/factory.h" #include "src/isolate-inl.h" #include "src/log-inl.h" +#include "src/trap-handler/trap-handler.h" #include "src/wasm/function-body-decoder.h" #include "src/wasm/wasm-limits.h" #include "src/wasm/wasm-module.h" @@ -82,6 +84,7 @@ WasmGraphBuilder::WasmGraphBuilder( function_table_sizes_(zone), cur_buffer_(def_buffer_), cur_bufsize_(kDefaultBufferSize), + untrusted_code_mitigations_(FLAG_untrusted_code_mitigations), runtime_exception_support_(exception_support), sig_(sig), source_position_table_(source_position_table) { @@ -118,10 +121,6 @@ Node* WasmGraphBuilder::Terminate(Node* effect, Node* control) { return terminate; } -unsigned WasmGraphBuilder::InputCount(Node* node) { - return static_cast<unsigned>(node->InputCount()); -} - bool WasmGraphBuilder::IsPhiWithMerge(Node* phi, Node* merge) { return phi && IrOpcode::IsPhiOpcode(phi->opcode()) && NodeProperties::GetControlInput(phi) == merge; @@ -2363,15 +2362,21 @@ Node* WasmGraphBuilder::BuildWasmCall(wasm::FunctionSig* sig, Node** args, Node* WasmGraphBuilder::CallDirect(uint32_t index, Node** args, Node*** rets, wasm::WasmCodePosition position) { DCHECK_NULL(args[0]); - - // Add code object as constant. - Handle<Code> code = index < env_->function_code.size() - ? env_->function_code[index] - : env_->default_function_code; - - DCHECK(!code.is_null()); - args[0] = HeapConstant(code); wasm::FunctionSig* sig = env_->module->functions[index].sig; + if (FLAG_wasm_jit_to_native) { + // Simply encode the index of the target. + Address code = reinterpret_cast<Address>(index); + args[0] = jsgraph()->RelocatableIntPtrConstant( + reinterpret_cast<intptr_t>(code), RelocInfo::WASM_CALL); + } else { + // Add code object as constant. + Handle<Code> code = index < env_->function_code.size() + ? env_->function_code[index] + : env_->default_function_code; + + DCHECK(!code.is_null()); + args[0] = HeapConstant(code); + } return BuildWasmCall(sig, args, rets, position); } @@ -2415,23 +2420,31 @@ Node* WasmGraphBuilder::CallIndirect(uint32_t sig_index, Node** args, Int32Constant(kPointerSizeLog2)), Int32Constant(fixed_offset)), *effect_, *control_); - auto map = env_->signature_maps[table_index]; - Node* sig_match = graph()->NewNode( - machine->WordEqual(), load_sig, - jsgraph()->SmiConstant(static_cast<int>(map->FindOrInsert(sig)))); + int32_t canonical_sig_num = env_->module->signature_ids[sig_index]; + CHECK_GE(sig_index, 0); + Node* sig_match = + graph()->NewNode(machine->WordEqual(), load_sig, + jsgraph()->SmiConstant(canonical_sig_num)); TrapIfFalse(wasm::kTrapFuncSigMismatch, sig_match, position); } - // Load code object from the table. - Node* load_code = graph()->NewNode( + // Load code object from the table. It is held by a Foreign. + Node* entry = graph()->NewNode( machine->Load(MachineType::AnyTagged()), table, graph()->NewNode(machine->Int32Add(), graph()->NewNode(machine->Word32Shl(), key, Int32Constant(kPointerSizeLog2)), Uint32Constant(fixed_offset)), *effect_, *control_); - - args[0] = load_code; + if (FLAG_wasm_jit_to_native) { + Node* address = graph()->NewNode( + machine->Load(MachineType::Pointer()), entry, + Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), + *effect_, *control_); + args[0] = address; + } else { + args[0] = entry; + } return BuildWasmCall(sig, args, rets, position); } @@ -2765,7 +2778,7 @@ Node* WasmGraphBuilder::BuildHeapNumberValueIndexConstant() { return jsgraph()->IntPtrConstant(HeapNumber::kValueOffset - kHeapObjectTag); } -void WasmGraphBuilder::BuildJSToWasmWrapper(Handle<Code> wasm_code, +void WasmGraphBuilder::BuildJSToWasmWrapper(WasmCodeWrapper wasm_code, Address wasm_context_address) { const int wasm_count = static_cast<int>(sig_->parameter_count()); const int count = @@ -2791,18 +2804,28 @@ void WasmGraphBuilder::BuildJSToWasmWrapper(Handle<Code> wasm_code, reinterpret_cast<uintptr_t>(wasm_context_address), RelocInfo::WASM_CONTEXT_REFERENCE); + Node* wasm_code_node = nullptr; + if (!wasm_code.IsCodeObject()) { + const wasm::WasmCode* code = wasm_code.GetWasmCode(); + Address instr_start = + code == nullptr ? nullptr : code->instructions().start(); + wasm_code_node = jsgraph()->RelocatableIntPtrConstant( + reinterpret_cast<intptr_t>(instr_start), RelocInfo::JS_TO_WASM_CALL); + } else { + wasm_code_node = HeapConstant(wasm_code.GetCode()); + } if (!wasm::IsJSCompatibleSignature(sig_)) { // Throw a TypeError. Use the js_context of the calling javascript function // (passed as a parameter), such that the generated code is js_context // independent. - BuildCallToRuntimeWithContextFromJS(Runtime::kWasmThrowTypeError, - js_context, nullptr, 0); + BuildCallToRuntimeWithContext(Runtime::kWasmThrowTypeError, js_context, + nullptr, 0); // Add a dummy call to the wasm function so that the generated wrapper // contains a reference to the wrapped wasm function. Without this reference // the wasm function could not be re-imported into another wasm module. int pos = 0; - args[pos++] = HeapConstant(wasm_code); + args[pos++] = wasm_code_node; args[pos++] = wasm_context_; args[pos++] = *effect_; args[pos++] = *control_; @@ -2817,7 +2840,7 @@ void WasmGraphBuilder::BuildJSToWasmWrapper(Handle<Code> wasm_code, } int pos = 0; - args[pos++] = HeapConstant(wasm_code); + args[pos++] = wasm_code_node; args[pos++] = wasm_context_; // Convert JS parameters to wasm numbers. @@ -3016,7 +3039,7 @@ bool HasInt64ParamOrReturn(wasm::FunctionSig* sig) { } } // namespace -void WasmGraphBuilder::BuildWasmToWasmWrapper(Handle<Code> target, +void WasmGraphBuilder::BuildWasmToWasmWrapper(WasmCodeWrapper wasm_code, Address new_context_address) { int wasm_count = static_cast<int>(sig_->parameter_count()); int count = wasm_count + 4; // wasm_code, wasm_context, effect, and control. @@ -3029,7 +3052,15 @@ void WasmGraphBuilder::BuildWasmToWasmWrapper(Handle<Code> target, int pos = 0; // Add the wasm code target. - args[pos++] = jsgraph()->HeapConstant(target); + if (!wasm_code.IsCodeObject()) { + const wasm::WasmCode* code = wasm_code.GetWasmCode(); + Address instr_start = + code == nullptr ? nullptr : code->instructions().start(); + args[pos++] = jsgraph()->RelocatableIntPtrConstant( + reinterpret_cast<intptr_t>(instr_start), RelocInfo::JS_TO_WASM_CALL); + } else { + args[pos++] = jsgraph()->HeapConstant(wasm_code.GetCode()); + } // Add the wasm_context of the other instance. args[pos++] = jsgraph()->IntPtrConstant( reinterpret_cast<uintptr_t>(new_context_address)); @@ -3041,16 +3072,14 @@ void WasmGraphBuilder::BuildWasmToWasmWrapper(Handle<Code> target, args[pos++] = *effect_; args[pos++] = *control_; - // Call the wasm code. + // Tail-call the wasm code. CallDescriptor* desc = GetWasmCallDescriptor(jsgraph()->zone(), sig_); - Node* call = graph()->NewNode(jsgraph()->common()->Call(desc), count, args); - *effect_ = call; - Node* retval = sig_->return_count() == 0 ? jsgraph()->Int32Constant(0) : call; - Return(retval); + Node* tail_call = + graph()->NewNode(jsgraph()->common()->TailCall(desc), count, args); + MergeControlToEnd(jsgraph(), tail_call); } -void WasmGraphBuilder::BuildWasmInterpreterEntry( - uint32_t function_index, Handle<WasmInstanceObject> instance) { +void WasmGraphBuilder::BuildWasmInterpreterEntry(uint32_t func_index) { int param_count = static_cast<int>(sig_->parameter_count()); // Build the start and the parameter nodes. @@ -3095,9 +3124,8 @@ void WasmGraphBuilder::BuildWasmInterpreterEntry( // like a Smi (lowest bit not set). In the runtime function however, don't // call Smi::value on it, but just cast it to a byte pointer. Node* parameters[] = { - jsgraph()->HeapConstant(instance), // wasm instance - jsgraph()->SmiConstant(function_index), // function index - arg_buffer, // argument buffer + jsgraph()->SmiConstant(func_index), // function index + arg_buffer, // argument buffer }; BuildCallToRuntime(Runtime::kWasmRunInterpreter, parameters, arraysize(parameters)); @@ -3130,12 +3158,19 @@ void WasmGraphBuilder::BuildCWasmEntry(Address wasm_context_address) { reinterpret_cast<uintptr_t>(wasm_context_address)); // Create parameter nodes (offset by 1 for the receiver parameter). - Node* code_obj = Param(CWasmEntryParameters::kCodeObject + 1); + Node* code_obj = nullptr; + if (FLAG_wasm_jit_to_native) { + Node* foreign_code_obj = Param(CWasmEntryParameters::kCodeObject + 1); + MachineOperatorBuilder* machine = jsgraph()->machine(); + code_obj = graph()->NewNode( + machine->Load(MachineType::Pointer()), foreign_code_obj, + Int32Constant(Foreign::kForeignAddressOffset - kHeapObjectTag), + *effect_, *control_); + } else { + code_obj = Param(CWasmEntryParameters::kCodeObject + 1); + } Node* arg_buffer = Param(CWasmEntryParameters::kArgumentsBuffer + 1); - // Set the ThreadInWasm flag before we do the actual call. - BuildModifyThreadInWasmFlag(true); - int wasm_arg_count = static_cast<int>(sig_->parameter_count()); int arg_count = wasm_arg_count + 4; // code, wasm_context, control, effect Node** args = Buffer(arg_count); @@ -3165,9 +3200,6 @@ void WasmGraphBuilder::BuildCWasmEntry(Address wasm_context_address) { graph()->NewNode(jsgraph()->common()->Call(desc), arg_count, args); *effect_ = call; - // Clear the ThreadInWasmFlag - BuildModifyThreadInWasmFlag(false); - // Store the return value. DCHECK_GE(1, sig_->return_count()); if (sig_->return_count() == 1) { @@ -3193,53 +3225,167 @@ void WasmGraphBuilder::BuildCWasmEntry(Address wasm_context_address) { } } -// This function is used by WasmFullDecoder to create a node that loads the -// mem_start variable from the WasmContext. It should not be used directly by -// the WasmGraphBuilder. The WasmGraphBuilder should directly use mem_start_, -// which will always contain the correct node (stored in the SsaEnv). -Node* WasmGraphBuilder::LoadMemStart() { +void WasmGraphBuilder::InitContextCache(WasmContextCacheNodes* context_cache) { DCHECK_NOT_NULL(wasm_context_); - Node* mem_buffer = graph()->NewNode( + DCHECK_NOT_NULL(*control_); + DCHECK_NOT_NULL(*effect_); + + // Load the memory start. + Node* mem_start = graph()->NewNode( jsgraph()->machine()->Load(MachineType::UintPtr()), wasm_context_, jsgraph()->Int32Constant( static_cast<int32_t>(offsetof(WasmContext, mem_start))), *effect_, *control_); - *effect_ = mem_buffer; - return mem_buffer; -} + *effect_ = mem_start; + context_cache->mem_start = mem_start; -// This function is used by WasmFullDecoder to create a node that loads the -// mem_size variable from the WasmContext. It should not be used directly by -// the WasmGraphBuilder. The WasmGraphBuilder should directly use mem_size_, -// which will always contain the correct node (stored in the SsaEnv). -Node* WasmGraphBuilder::LoadMemSize() { - // Load mem_size from the memory_context location at runtime. - DCHECK_NOT_NULL(wasm_context_); + // Load the memory size. Node* mem_size = graph()->NewNode( jsgraph()->machine()->Load(MachineType::Uint32()), wasm_context_, jsgraph()->Int32Constant( static_cast<int32_t>(offsetof(WasmContext, mem_size))), *effect_, *control_); *effect_ = mem_size; - if (jsgraph()->machine()->Is64()) { - mem_size = graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), - mem_size); + context_cache->mem_size = mem_size; + + if (untrusted_code_mitigations_) { + // Load the memory mask. + Node* mem_mask = graph()->NewNode( + jsgraph()->machine()->Load(MachineType::Uint32()), wasm_context_, + jsgraph()->Int32Constant( + static_cast<int32_t>(offsetof(WasmContext, mem_mask))), + *effect_, *control_); + *effect_ = mem_mask; + context_cache->mem_mask = mem_mask; + } else { + // Explicitly set to nullptr to ensure a SEGV when we try to use it. + context_cache->mem_mask = nullptr; + } +} + +void WasmGraphBuilder::PrepareContextCacheForLoop( + WasmContextCacheNodes* context_cache, Node* control) { +#define INTRODUCE_PHI(field, rep) \ + context_cache->field = Phi(rep, 1, &context_cache->field, control); + + INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); + INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); + if (untrusted_code_mitigations_) { + INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); + } + +#undef INTRODUCE_PHI +} + +void WasmGraphBuilder::NewContextCacheMerge(WasmContextCacheNodes* to, + WasmContextCacheNodes* from, + Node* merge) { +#define INTRODUCE_PHI(field, rep) \ + if (to->field != from->field) { \ + Node* vals[] = {to->field, from->field}; \ + to->field = Phi(rep, 2, vals, merge); \ + } + + INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); + INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); + if (untrusted_code_mitigations_) { + INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); + } + +#undef INTRODUCE_PHI +} + +void WasmGraphBuilder::MergeContextCacheInto(WasmContextCacheNodes* to, + WasmContextCacheNodes* from, + Node* merge) { + to->mem_size = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge, + to->mem_size, from->mem_size); + to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), + merge, to->mem_start, from->mem_start); + if (untrusted_code_mitigations_) { + to->mem_mask = CreateOrMergeIntoPhi(MachineRepresentation::kWord32, merge, + to->mem_mask, from->mem_mask); + } +} + +Node* WasmGraphBuilder::CreateOrMergeIntoPhi(wasm::ValueType type, Node* merge, + Node* tnode, Node* fnode) { + if (IsPhiWithMerge(tnode, merge)) { + AppendToPhi(tnode, fnode); + } else if (tnode != fnode) { + uint32_t count = merge->InputCount(); + Node** vals = Buffer(count); + for (uint32_t j = 0; j < count - 1; j++) vals[j] = tnode; + vals[count - 1] = fnode; + return Phi(type, count, vals, merge); + } + return tnode; +} + +Node* WasmGraphBuilder::CreateOrMergeIntoEffectPhi(Node* merge, Node* tnode, + Node* fnode) { + if (IsPhiWithMerge(tnode, merge)) { + AppendToPhi(tnode, fnode); + } else if (tnode != fnode) { + uint32_t count = merge->InputCount(); + Node** effects = Buffer(count); + for (uint32_t j = 0; j < count - 1; j++) { + effects[j] = tnode; + } + effects[count - 1] = fnode; + tnode = EffectPhi(count, effects, merge); + } + return tnode; +} + +void WasmGraphBuilder::GetGlobalBaseAndOffset(MachineType mem_type, + uint32_t offset, Node** base_node, + Node** offset_node) { + DCHECK_NOT_NULL(wasm_context_); + if (globals_start_ == nullptr) { + // Load globals_start from the WasmContext at runtime. + // TODO(wasm): we currently generate only one load of the {globals_start} + // start per graph, which means it can be placed anywhere by the scheduler. + // This is legal because the globals_start should never change. + // However, in some cases (e.g. if the WasmContext is already in a + // register), it is slightly more efficient to reload this value from the + // WasmContext. Since this depends on register allocation, it is not + // possible to express in the graph, and would essentially constitute a + // "mem2reg" optimization in TurboFan. + globals_start_ = graph()->NewNode( + jsgraph()->machine()->Load(MachineType::UintPtr()), wasm_context_, + jsgraph()->Int32Constant( + static_cast<int32_t>(offsetof(WasmContext, globals_start))), + graph()->start(), graph()->start()); + } + *base_node = globals_start_; + *offset_node = jsgraph()->Int32Constant(offset); + + if (mem_type == MachineType::Simd128() && offset != 0) { + // TODO(titzer,bbudge): code generation for SIMD memory offsets is broken. + *base_node = + graph()->NewNode(kPointerSize == 4 ? jsgraph()->machine()->Int32Add() + : jsgraph()->machine()->Int64Add(), + *base_node, *offset_node); + *offset_node = jsgraph()->Int32Constant(0); } - return mem_size; } Node* WasmGraphBuilder::MemBuffer(uint32_t offset) { - DCHECK_NOT_NULL(*mem_start_); - if (offset == 0) return *mem_start_; - return graph()->NewNode(jsgraph()->machine()->IntAdd(), *mem_start_, + DCHECK_NOT_NULL(context_cache_); + Node* mem_start = context_cache_->mem_start; + DCHECK_NOT_NULL(mem_start); + if (offset == 0) return mem_start; + return graph()->NewNode(jsgraph()->machine()->IntAdd(), mem_start, jsgraph()->IntPtrConstant(offset)); } Node* WasmGraphBuilder::CurrentMemoryPages() { // CurrentMemoryPages can not be called from asm.js. DCHECK_EQ(wasm::kWasmOrigin, env_->module->origin()); - DCHECK_NOT_NULL(*mem_size_); - Node* mem_size = *mem_size_; + DCHECK_NOT_NULL(context_cache_); + Node* mem_size = context_cache_->mem_size; + DCHECK_NOT_NULL(mem_size); if (jsgraph()->machine()->Is64()) { mem_size = graph()->NewNode(jsgraph()->machine()->TruncateInt64ToInt32(), mem_size); @@ -3296,23 +3442,6 @@ Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f, Node* js_context, Node** parameters, int parameter_count) { - // We're leaving Wasm code, so clear the flag. - *control_ = BuildModifyThreadInWasmFlag(false); - // Since the thread-in-wasm flag is clear, it is as if we are calling from JS. - Node* call = BuildCallToRuntimeWithContextFromJS(f, js_context, parameters, - parameter_count); - - // Restore the thread-in-wasm flag, since we have returned to Wasm. - *control_ = BuildModifyThreadInWasmFlag(true); - - return call; -} - -// This version of BuildCallToRuntime does not clear and set the thread-in-wasm -// flag. -Node* WasmGraphBuilder::BuildCallToRuntimeWithContextFromJS( - Runtime::FunctionId f, Node* js_context, Node* const* parameters, - int parameter_count) { const Runtime::Function* fun = Runtime::FunctionForId(f); CallDescriptor* desc = Linkage::GetRuntimeCallDescriptor( jsgraph()->zone(), f, fun->nargs, Operator::kNoProperties, @@ -3354,13 +3483,12 @@ Node* WasmGraphBuilder::BuildCallToRuntime(Runtime::FunctionId f, Node* WasmGraphBuilder::GetGlobal(uint32_t index) { MachineType mem_type = wasm::WasmOpcodes::MachineTypeFor(env_->module->globals[index].type); - uintptr_t global_addr = - env_->globals_start + env_->module->globals[index].offset; - Node* addr = jsgraph()->RelocatableIntPtrConstant( - global_addr, RelocInfo::WASM_GLOBAL_REFERENCE); - const Operator* op = jsgraph()->machine()->Load(mem_type); - Node* node = graph()->NewNode(op, addr, jsgraph()->Int32Constant(0), *effect_, - *control_); + Node* base = nullptr; + Node* offset = nullptr; + GetGlobalBaseAndOffset(mem_type, env_->module->globals[index].offset, &base, + &offset); + Node* node = graph()->NewNode(jsgraph()->machine()->Load(mem_type), base, + offset, *effect_, *control_); *effect_ = node; return node; } @@ -3368,23 +3496,33 @@ Node* WasmGraphBuilder::GetGlobal(uint32_t index) { Node* WasmGraphBuilder::SetGlobal(uint32_t index, Node* val) { MachineType mem_type = wasm::WasmOpcodes::MachineTypeFor(env_->module->globals[index].type); - uintptr_t global_addr = - env_->globals_start + env_->module->globals[index].offset; - Node* addr = jsgraph()->RelocatableIntPtrConstant( - global_addr, RelocInfo::WASM_GLOBAL_REFERENCE); + Node* base = nullptr; + Node* offset = nullptr; + GetGlobalBaseAndOffset(mem_type, env_->module->globals[index].offset, &base, + &offset); const Operator* op = jsgraph()->machine()->Store( StoreRepresentation(mem_type.representation(), kNoWriteBarrier)); - Node* node = graph()->NewNode(op, addr, jsgraph()->Int32Constant(0), val, - *effect_, *control_); + Node* node = graph()->NewNode(op, base, offset, val, *effect_, *control_); *effect_ = node; return node; } -void WasmGraphBuilder::BoundsCheckMem(MachineType memtype, Node* index, - uint32_t offset, - wasm::WasmCodePosition position) { - if (FLAG_wasm_no_bounds_checks) return; - DCHECK_NOT_NULL(*mem_size_); +Node* WasmGraphBuilder::BoundsCheckMem(MachineType memtype, Node* index, + uint32_t offset, + wasm::WasmCodePosition position, + EnforceBoundsCheck enforce_check) { + if (FLAG_wasm_no_bounds_checks) return index; + DCHECK_NOT_NULL(context_cache_); + Node* mem_size = context_cache_->mem_size; + DCHECK_NOT_NULL(mem_size); + + auto m = jsgraph()->machine(); + if (trap_handler::UseTrapHandler() && enforce_check == kCanOmitBoundsCheck) { + // Simply zero out the 32-bits on 64-bit targets and let the trap handler + // do its job. + return m->Is64() ? graph()->NewNode(m->ChangeUint32ToUint64(), index) + : index; + } uint32_t min_size = env_->module->initial_pages * wasm::WasmModule::kPageSize; uint32_t max_size = @@ -3398,7 +3536,7 @@ void WasmGraphBuilder::BoundsCheckMem(MachineType memtype, Node* index, // The access will be out of bounds, even for the largest memory. TrapIfEq32(wasm::kTrapMemOutOfBounds, jsgraph()->Int32Constant(0), 0, position); - return; + return jsgraph()->IntPtrConstant(0); } uint32_t end_offset = offset + access_size; @@ -3406,48 +3544,44 @@ void WasmGraphBuilder::BoundsCheckMem(MachineType memtype, Node* index, // The end offset is larger than the smallest memory. // Dynamically check the end offset against the actual memory size, which // is not known at compile time. - Node* cond; - if (jsgraph()->machine()->Is32()) { - cond = graph()->NewNode(jsgraph()->machine()->Uint32LessThanOrEqual(), - jsgraph()->Int32Constant(end_offset), *mem_size_); - } else { - cond = graph()->NewNode( - jsgraph()->machine()->Uint64LessThanOrEqual(), - jsgraph()->Int64Constant(static_cast<int64_t>(end_offset)), - *mem_size_); - } + Node* cond = + graph()->NewNode(jsgraph()->machine()->Uint32LessThanOrEqual(), + jsgraph()->Int32Constant(end_offset), mem_size); TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); } else { // The end offset is within the bounds of the smallest memory, so only // one check is required. Check to see if the index is also a constant. - UintPtrMatcher m(index); - if (m.HasValue()) { - uint64_t index_val = m.Value(); - if ((index_val + offset + access_size) <= min_size) { + Uint32Matcher match(index); + if (match.HasValue()) { + uint32_t index_val = match.Value(); + if (index_val <= min_size - end_offset) { // The input index is a constant and everything is statically within // bounds of the smallest possible memory. - return; + return m->Is64() ? graph()->NewNode(m->ChangeUint32ToUint64(), index) + : index; } } } - Node* effective_size; - if (jsgraph()->machine()->Is32()) { - effective_size = - graph()->NewNode(jsgraph()->machine()->Int32Sub(), *mem_size_, - jsgraph()->Int32Constant(end_offset - 1)); - } else { - effective_size = graph()->NewNode( - jsgraph()->machine()->Int64Sub(), *mem_size_, - jsgraph()->Int64Constant(static_cast<int64_t>(end_offset - 1))); - } - - const Operator* less = jsgraph()->machine()->Is32() - ? jsgraph()->machine()->Uint32LessThan() - : jsgraph()->machine()->Uint64LessThan(); + // Compute the effective size of the memory, which is the size of the memory + // minus the statically known offset, minus the byte size of the access minus + // one. + // This produces a positive number since {end_offset <= min_size <= mem_size}. + Node* effective_size = + graph()->NewNode(jsgraph()->machine()->Int32Sub(), mem_size, + jsgraph()->Int32Constant(end_offset - 1)); - Node* cond = graph()->NewNode(less, index, effective_size); + // Introduce the actual bounds check. + Node* cond = graph()->NewNode(m->Uint32LessThan(), index, effective_size); TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); + + if (untrusted_code_mitigations_) { + // In the fallthrough case, condition the index with the memory mask. + Node* mem_mask = context_cache_->mem_mask; + DCHECK_NOT_NULL(mem_mask); + index = graph()->NewNode(m->Word32And(), index, mem_mask); + } + return m->Is64() ? graph()->NewNode(m->ChangeUint32ToUint64(), index) : index; } const Operator* WasmGraphBuilder::GetSafeLoadOperator(int offset, @@ -3500,14 +3634,9 @@ Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype, wasm::WasmCodePosition position) { Node* load; - if (jsgraph()->machine()->Is64()) { - index = - graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), index); - } - // Wasm semantics throw on OOB. Introduce explicit bounds check. - if (!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED) { - BoundsCheckMem(memtype, index, offset, position); - } + // Wasm semantics throw on OOB. Introduce explicit bounds check and + // conditioning when not using the trap handler. + index = BoundsCheckMem(memtype, index, offset, position, kCanOmitBoundsCheck); if (memtype.representation() == MachineRepresentation::kWord8 || jsgraph()->machine()->UnalignedLoadSupported(memtype.representation())) { @@ -3521,7 +3650,7 @@ Node* WasmGraphBuilder::LoadMem(wasm::ValueType type, MachineType memtype, } } else { // TODO(eholk): Support unaligned loads with trap handlers. - DCHECK(!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED); + DCHECK(!trap_handler::UseTrapHandler()); load = graph()->NewNode(jsgraph()->machine()->UnalignedLoad(memtype), MemBuffer(offset), index, *effect_, *control_); } @@ -3559,14 +3688,7 @@ Node* WasmGraphBuilder::StoreMem(MachineType memtype, Node* index, wasm::ValueType type) { Node* store; - if (jsgraph()->machine()->Is64()) { - index = - graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), index); - } - // Wasm semantics throw on OOB. Introduce explicit bounds check. - if (!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED) { - BoundsCheckMem(memtype, index, offset, position); - } + index = BoundsCheckMem(memtype, index, offset, position, kCanOmitBoundsCheck); #if defined(V8_TARGET_BIG_ENDIAN) val = BuildChangeEndiannessStore(val, memtype, type); @@ -3574,7 +3696,7 @@ Node* WasmGraphBuilder::StoreMem(MachineType memtype, Node* index, if (memtype.representation() == MachineRepresentation::kWord8 || jsgraph()->machine()->UnalignedStoreSupported(memtype.representation())) { - if (FLAG_wasm_trap_handler && V8_TRAP_HANDLER_SUPPORTED) { + if (trap_handler::UseTrapHandler()) { store = graph()->NewNode( jsgraph()->machine()->ProtectedStore(memtype.representation()), MemBuffer(offset), index, val, *effect_, *control_); @@ -3587,7 +3709,7 @@ Node* WasmGraphBuilder::StoreMem(MachineType memtype, Node* index, } } else { // TODO(eholk): Support unaligned stores with trap handlers. - DCHECK(!FLAG_wasm_trap_handler || !V8_TRAP_HANDLER_SUPPORTED); + DCHECK(!trap_handler::UseTrapHandler()); UnalignedStoreRepresentation rep(memtype.representation()); store = graph()->NewNode(jsgraph()->machine()->UnalignedStore(rep), @@ -3604,37 +3726,106 @@ Node* WasmGraphBuilder::StoreMem(MachineType memtype, Node* index, return store; } +namespace { +Node* GetAsmJsOOBValue(MachineRepresentation rep, JSGraph* jsgraph) { + switch (rep) { + case MachineRepresentation::kWord8: + case MachineRepresentation::kWord16: + case MachineRepresentation::kWord32: + return jsgraph->Int32Constant(0); + case MachineRepresentation::kWord64: + return jsgraph->Int64Constant(0); + case MachineRepresentation::kFloat32: + return jsgraph->Float32Constant(std::numeric_limits<float>::quiet_NaN()); + case MachineRepresentation::kFloat64: + return jsgraph->Float64Constant(std::numeric_limits<double>::quiet_NaN()); + default: + UNREACHABLE(); + } +} +} // namespace + Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) { - // TODO(turbofan): fold bounds checks for constant asm.js loads. - // asm.js semantics use CheckedLoad (i.e. OOB reads return 0ish). - DCHECK_NOT_NULL(*mem_size_); - DCHECK_NOT_NULL(*mem_start_); + DCHECK_NOT_NULL(context_cache_); + Node* mem_start = context_cache_->mem_start; + Node* mem_size = context_cache_->mem_size; + DCHECK_NOT_NULL(mem_start); + DCHECK_NOT_NULL(mem_size); + + // Asm.js semantics are defined along the lines of typed arrays, hence OOB + // reads return {undefined} coerced to the result type (0 for integers, NaN + // for float and double). + // Note that we check against the memory size ignoring the size of the + // stored value, which is conservative if misaligned. Technically, asm.js + // should never have misaligned accesses. + Diamond bounds_check( + graph(), jsgraph()->common(), + graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index, mem_size), + BranchHint::kTrue); + bounds_check.Chain(*control_); + + if (untrusted_code_mitigations_) { + // Condition the index with the memory mask. + Node* mem_mask = context_cache_->mem_mask; + DCHECK_NOT_NULL(mem_mask); + index = + graph()->NewNode(jsgraph()->machine()->Word32And(), index, mem_mask); + } + if (jsgraph()->machine()->Is64()) { index = graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), index); } - const Operator* op = jsgraph()->machine()->CheckedLoad(type); - Node* load = - graph()->NewNode(op, *mem_start_, index, *mem_size_, *effect_, *control_); - *effect_ = load; - return load; + Node* load = graph()->NewNode(jsgraph()->machine()->Load(type), mem_start, + index, *effect_, bounds_check.if_true); + Node* value_phi = + bounds_check.Phi(type.representation(), load, + GetAsmJsOOBValue(type.representation(), jsgraph())); + Node* effect_phi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), load, + *effect_, bounds_check.merge); + *effect_ = effect_phi; + *control_ = bounds_check.merge; + return value_phi; } Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index, Node* val) { - // TODO(turbofan): fold bounds checks for constant asm.js stores. - // asm.js semantics use CheckedStore (i.e. ignore OOB writes). - DCHECK_NOT_NULL(*mem_size_); - DCHECK_NOT_NULL(*mem_start_); + DCHECK_NOT_NULL(context_cache_); + Node* mem_start = context_cache_->mem_start; + Node* mem_size = context_cache_->mem_size; + DCHECK_NOT_NULL(mem_start); + DCHECK_NOT_NULL(mem_size); + + // Asm.js semantics are to ignore OOB writes. + // Note that we check against the memory size ignoring the size of the + // stored value, which is conservative if misaligned. Technically, asm.js + // should never have misaligned accesses. + Diamond bounds_check( + graph(), jsgraph()->common(), + graph()->NewNode(jsgraph()->machine()->Uint32LessThan(), index, mem_size), + BranchHint::kTrue); + bounds_check.Chain(*control_); + + if (untrusted_code_mitigations_) { + // Condition the index with the memory mask. + Node* mem_mask = context_cache_->mem_mask; + DCHECK_NOT_NULL(mem_mask); + index = + graph()->NewNode(jsgraph()->machine()->Word32And(), index, mem_mask); + } + if (jsgraph()->machine()->Is64()) { index = graph()->NewNode(jsgraph()->machine()->ChangeUint32ToUint64(), index); } - const Operator* op = - jsgraph()->machine()->CheckedStore(type.representation()); - Node* store = graph()->NewNode(op, *mem_start_, index, *mem_size_, val, - *effect_, *control_); - *effect_ = store; + const Operator* store_op = jsgraph()->machine()->Store(StoreRepresentation( + type.representation(), WriteBarrierKind::kNoWriteBarrier)); + Node* store = graph()->NewNode(store_op, mem_start, index, val, *effect_, + bounds_check.if_true); + Node* effect_phi = graph()->NewNode(jsgraph()->common()->EffectPhi(2), store, + *effect_, bounds_check.merge); + *effect_ = effect_phi; + *control_ = bounds_check.merge; return val; } @@ -4111,47 +4302,50 @@ Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs, // TODO(gdeepti): Add alignment validation, traps on misalignment Node* node; switch (opcode) { -#define BUILD_ATOMIC_BINOP(Name, Operation, Type) \ - case wasm::kExpr##Name: { \ - BoundsCheckMem(MachineType::Type(), inputs[0], offset, position); \ - node = graph()->NewNode( \ - jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ - MemBuffer(offset), inputs[0], inputs[1], *effect_, *control_); \ - break; \ +#define BUILD_ATOMIC_BINOP(Name, Operation, Type) \ + case wasm::kExpr##Name: { \ + Node* index = BoundsCheckMem(MachineType::Type(), inputs[0], offset, \ + position, kNeedsBoundsCheck); \ + node = graph()->NewNode( \ + jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ + MemBuffer(offset), index, inputs[1], *effect_, *control_); \ + break; \ } ATOMIC_BINOP_LIST(BUILD_ATOMIC_BINOP) #undef BUILD_ATOMIC_BINOP -#define BUILD_ATOMIC_TERNARY_OP(Name, Operation, Type) \ - case wasm::kExpr##Name: { \ - BoundsCheckMem(MachineType::Type(), inputs[0], offset, position); \ - node = graph()->NewNode( \ - jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ - MemBuffer(offset), inputs[0], inputs[1], inputs[2], *effect_, \ - *control_); \ - break; \ +#define BUILD_ATOMIC_TERNARY_OP(Name, Operation, Type) \ + case wasm::kExpr##Name: { \ + Node* index = BoundsCheckMem(MachineType::Type(), inputs[0], offset, \ + position, kNeedsBoundsCheck); \ + node = graph()->NewNode( \ + jsgraph()->machine()->Atomic##Operation(MachineType::Type()), \ + MemBuffer(offset), index, inputs[1], inputs[2], *effect_, *control_); \ + break; \ } ATOMIC_TERNARY_LIST(BUILD_ATOMIC_TERNARY_OP) #undef BUILD_ATOMIC_TERNARY_OP -#define BUILD_ATOMIC_LOAD_OP(Name, Type) \ - case wasm::kExpr##Name: { \ - BoundsCheckMem(MachineType::Type(), inputs[0], offset, position); \ - node = graph()->NewNode( \ - jsgraph()->machine()->AtomicLoad(MachineType::Type()), \ - MemBuffer(offset), inputs[0], *effect_, *control_); \ - break; \ +#define BUILD_ATOMIC_LOAD_OP(Name, Type) \ + case wasm::kExpr##Name: { \ + Node* index = BoundsCheckMem(MachineType::Type(), inputs[0], offset, \ + position, kNeedsBoundsCheck); \ + node = graph()->NewNode( \ + jsgraph()->machine()->AtomicLoad(MachineType::Type()), \ + MemBuffer(offset), index, *effect_, *control_); \ + break; \ } ATOMIC_LOAD_LIST(BUILD_ATOMIC_LOAD_OP) #undef BUILD_ATOMIC_LOAD_OP -#define BUILD_ATOMIC_STORE_OP(Name, Type, Rep) \ - case wasm::kExpr##Name: { \ - BoundsCheckMem(MachineType::Type(), inputs[0], offset, position); \ - node = graph()->NewNode( \ - jsgraph()->machine()->AtomicStore(MachineRepresentation::Rep), \ - MemBuffer(offset), inputs[0], inputs[1], *effect_, *control_); \ - break; \ +#define BUILD_ATOMIC_STORE_OP(Name, Type, Rep) \ + case wasm::kExpr##Name: { \ + Node* index = BoundsCheckMem(MachineType::Type(), inputs[0], offset, \ + position, kNeedsBoundsCheck); \ + node = graph()->NewNode( \ + jsgraph()->machine()->AtomicStore(MachineRepresentation::Rep), \ + MemBuffer(offset), index, inputs[1], *effect_, *control_); \ + break; \ } ATOMIC_STORE_LIST(BUILD_ATOMIC_STORE_OP) #undef BUILD_ATOMIC_STORE_OP @@ -4196,7 +4390,7 @@ void RecordFunctionCompilation(CodeEventListener::LogEventsAndTags tag, } // namespace Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, - Handle<Code> wasm_code, uint32_t index, + WasmCodeWrapper wasm_code, uint32_t index, Address wasm_context_address) { const wasm::WasmFunction* func = &module->functions[index]; @@ -4206,20 +4400,25 @@ Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); - MachineOperatorBuilder machine(&zone); + MachineOperatorBuilder machine( + &zone, MachineType::PointerRepresentation(), + InstructionSelector::SupportedMachineOperatorFlags(), + InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; Node* effect = nullptr; // TODO(titzer): compile JS to WASM wrappers without a {ModuleEnv}. - ModuleEnv env = {module, - std::vector<Address>(), // function_tables - std::vector<Address>(), // signature_tables - std::vector<wasm::SignatureMap*>(), // signature_maps - std::vector<Handle<Code>>(), // function_code - BUILTIN_CODE(isolate, Illegal), // default_function_code - 0}; + ModuleEnv env = { + module, + std::vector<Address>(), // function_tables + std::vector<Address>(), // signature_tables + // TODO(mtrofin): remove these 2 lines when we don't need + // FLAG_wasm_jit_to_native + std::vector<Handle<Code>>(), // function_code + BUILTIN_CODE(isolate, Illegal) // default_function_code + }; WasmGraphBuilder builder(&env, &zone, &jsgraph, CEntryStub(isolate, 1).GetCode(), func->sig); @@ -4250,8 +4449,9 @@ Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, Vector<const char> func_name = CStrVector("js-to-wasm"); #endif - CompilationInfo info(func_name, isolate, &zone, Code::JS_TO_WASM_FUNCTION); - Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph); + CompilationInfo info(func_name, &zone, Code::JS_TO_WASM_FUNCTION); + Handle<Code> code = + Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); @@ -4267,6 +4467,8 @@ Handle<Code> CompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, return code; } +namespace { + void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) { #if !DEBUG return; @@ -4283,7 +4485,7 @@ void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) { case RelocInfo::CODE_TARGET: // this would be either one of the stubs or builtins, because // we didn't link yet. - target = reinterpret_cast<Object*>(it.rinfo()->target_address()); + target = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); break; case RelocInfo::EMBEDDED_OBJECT: target = it.rinfo()->target_object(); @@ -4294,10 +4496,19 @@ void ValidateImportWrapperReferencesImmovables(Handle<Code> wrapper) { CHECK_NOT_NULL(target); bool is_immovable = target->IsSmi() || Heap::IsImmovable(HeapObject::cast(target)); - CHECK(is_immovable); + bool is_allowed_stub = false; + if (target->IsCode()) { + Code* code = Code::cast(target); + is_allowed_stub = + code->kind() == Code::STUB && + CodeStub::MajorKeyFromKey(code->stub_key()) == CodeStub::DoubleToI; + } + CHECK(is_immovable || is_allowed_stub); } } +} // namespace + Handle<Code> CompileWasmToJSWrapper( Isolate* isolate, Handle<JSReceiver> target, wasm::FunctionSig* sig, uint32_t index, wasm::ModuleOrigin origin, @@ -4308,7 +4519,10 @@ Handle<Code> CompileWasmToJSWrapper( Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); - MachineOperatorBuilder machine(&zone); + MachineOperatorBuilder machine( + &zone, MachineType::PointerRepresentation(), + InstructionSelector::SupportedMachineOperatorFlags(), + InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; @@ -4357,9 +4571,9 @@ Handle<Code> CompileWasmToJSWrapper( Vector<const char> func_name = CStrVector("wasm-to-js"); #endif - CompilationInfo info(func_name, isolate, &zone, Code::WASM_TO_JS_FUNCTION); + CompilationInfo info(func_name, &zone, Code::WASM_TO_JS_FUNCTION); Handle<Code> code = Pipeline::GenerateCodeForTesting( - &info, incoming, &graph, nullptr, source_position_table); + &info, isolate, incoming, &graph, nullptr, source_position_table); ValidateImportWrapperReferencesImmovables(code); Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(2, TENURED); @@ -4386,8 +4600,8 @@ Handle<Code> CompileWasmToJSWrapper( return code; } -Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, Handle<Code> target, - wasm::FunctionSig* sig, uint32_t index, +Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, WasmCodeWrapper target, + wasm::FunctionSig* sig, Address new_wasm_context_address) { //---------------------------------------------------------------------------- // Create the Graph @@ -4395,7 +4609,10 @@ Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, Handle<Code> target, Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); - MachineOperatorBuilder machine(&zone); + MachineOperatorBuilder machine( + &zone, MachineType::PointerRepresentation(), + InstructionSelector::SupportedMachineOperatorFlags(), + InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; @@ -4433,8 +4650,9 @@ Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, Handle<Code> target, func_name = Vector<const char>::cast(buffer.SubVector(0, chars)); } - CompilationInfo info(func_name, isolate, &zone, Code::WASM_FUNCTION); - Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph); + CompilationInfo info(func_name, &zone, Code::WASM_TO_WASM_FUNCTION); + Handle<Code> code = + Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); @@ -4446,7 +4664,7 @@ Handle<Code> CompileWasmToWasmWrapper(Isolate* isolate, Handle<Code> target, } if (isolate->logger()->is_logging_code_events() || isolate->is_profiling()) { RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate, code, - "wasm-to-wasm#%d", index); + "wasm-to-wasm"); } return code; @@ -4474,7 +4692,7 @@ Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index, CEntryStub(isolate, 1).GetCode(), sig); builder.set_control_ptr(&control); builder.set_effect_ptr(&effect); - builder.BuildWasmInterpreterEntry(func_index, instance); + builder.BuildWasmInterpreterEntry(func_index); Handle<Code> code = Handle<Code>::null(); { @@ -4497,9 +4715,9 @@ Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index, Vector<const char> func_name = CStrVector("wasm-interpreter-entry"); #endif - CompilationInfo info(func_name, isolate, &zone, - Code::WASM_INTERPRETER_ENTRY); - code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph, nullptr); + CompilationInfo info(func_name, &zone, Code::WASM_INTERPRETER_ENTRY); + code = Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph, + nullptr); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); @@ -4513,11 +4731,13 @@ Handle<Code> CompileWasmInterpreterEntry(Isolate* isolate, uint32_t func_index, } } - Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(1, TENURED); - Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance); - deopt_data->set(0, *weak_instance); - code->set_deoptimization_data(*deopt_data); - + if (!FLAG_wasm_jit_to_native) { + Handle<FixedArray> deopt_data = + isolate->factory()->NewFixedArray(1, TENURED); + Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance); + deopt_data->set(0, *weak_instance); + code->set_deoptimization_data(*deopt_data); + } return code; } @@ -4526,7 +4746,10 @@ Handle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig, Zone zone(isolate->allocator(), ZONE_NAME); Graph graph(&zone); CommonOperatorBuilder common(&zone); - MachineOperatorBuilder machine(&zone); + MachineOperatorBuilder machine( + &zone, MachineType::PointerRepresentation(), + InstructionSelector::SupportedMachineOperatorFlags(), + InstructionSelector::AlignmentRequirements()); JSGraph jsgraph(isolate, &graph, &common, nullptr, nullptr, &machine); Node* control = nullptr; @@ -4566,8 +4789,9 @@ Handle<Code> CompileCWasmEntry(Isolate* isolate, wasm::FunctionSig* sig, debug_name[name_len] = '\0'; Vector<const char> debug_name_vec(debug_name, name_len); - CompilationInfo info(debug_name_vec, isolate, &zone, Code::C_WASM_ENTRY); - Handle<Code> code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph); + CompilationInfo info(debug_name_vec, &zone, Code::C_WASM_ENTRY); + Handle<Code> code = + Pipeline::GenerateCodeForTesting(&info, isolate, incoming, &graph); #ifdef ENABLE_DISASSEMBLER if (FLAG_print_opt_code && !code.is_null()) { OFStream os(stdout); @@ -4585,7 +4809,6 @@ SourcePositionTable* WasmCompilationUnit::BuildGraphForWasmFunction( size_t tables_size = env_->module->function_tables.size(); DCHECK_EQ(tables_size, env_->function_tables.size()); DCHECK_EQ(tables_size, env_->signature_tables.size()); - DCHECK_EQ(tables_size, env_->signature_maps.size()); } #endif @@ -4596,17 +4819,17 @@ SourcePositionTable* WasmCompilationUnit::BuildGraphForWasmFunction( // Create a TF graph during decoding. SourcePositionTable* source_position_table = - new (jsgraph_->zone()) SourcePositionTable(jsgraph_->graph()); - WasmGraphBuilder builder(env_, jsgraph_->zone(), jsgraph_, centry_stub_, - func_body_.sig, source_position_table, + new (tf_.jsgraph_->zone()) SourcePositionTable(tf_.jsgraph_->graph()); + WasmGraphBuilder builder(env_, tf_.jsgraph_->zone(), tf_.jsgraph_, + centry_stub_, func_body_.sig, source_position_table, runtime_exception_support_); - graph_construction_result_ = + tf_.graph_construction_result_ = wasm::BuildTFGraph(isolate_->allocator(), &builder, func_body_); - if (graph_construction_result_.failed()) { + if (tf_.graph_construction_result_.failed()) { if (FLAG_trace_wasm_compiler) { OFStream os(stdout); - os << "Compilation failed: " << graph_construction_result_.error_msg() + os << "Compilation failed: " << tf_.graph_construction_result_.error_msg() << std::endl; } return nullptr; @@ -4616,7 +4839,7 @@ SourcePositionTable* WasmCompilationUnit::BuildGraphForWasmFunction( if (builder.has_simd() && (!CpuFeatures::SupportsWasmSimd128() || lower_simd_)) { - SimdScalarLowering(jsgraph_, func_body_.sig).LowerGraph(); + SimdScalarLowering(tf_.jsgraph_, func_body_.sig).LowerGraph(); } if (func_index_ >= FLAG_trace_wasm_ast_start && @@ -4647,13 +4870,21 @@ Vector<const char> GetDebugName(Zone* zone, wasm::WasmName name, int index) { memcpy(index_name, name_vector.start(), name_len); return Vector<const char>(index_name, name_len); } + } // namespace +// static +WasmCompilationUnit::CompilationMode +WasmCompilationUnit::GetDefaultCompilationMode() { + return FLAG_liftoff ? WasmCompilationUnit::CompilationMode::kLiftoff + : WasmCompilationUnit::CompilationMode::kTurbofan; +} + WasmCompilationUnit::WasmCompilationUnit( - Isolate* isolate, ModuleEnv* env, wasm::FunctionBody body, - wasm::WasmName name, int index, Handle<Code> centry_stub, - Counters* counters, RuntimeExceptionSupport exception_support, - bool lower_simd) + Isolate* isolate, ModuleEnv* env, wasm::NativeModule* native_module, + wasm::FunctionBody body, wasm::WasmName name, int index, + Handle<Code> centry_stub, CompilationMode mode, Counters* counters, + RuntimeExceptionSupport exception_support, bool lower_simd) : isolate_(isolate), env_(env), func_body_(body), @@ -4662,9 +4893,38 @@ WasmCompilationUnit::WasmCompilationUnit( centry_stub_(centry_stub), func_index_(index), runtime_exception_support_(exception_support), - lower_simd_(lower_simd) {} + native_module_(native_module), + lower_simd_(lower_simd), + protected_instructions_( + new std::vector<trap_handler::ProtectedInstructionData>()), + mode_(mode) { + switch (mode_) { + case WasmCompilationUnit::CompilationMode::kLiftoff: + new (&liftoff_) LiftoffData(isolate); + break; + case WasmCompilationUnit::CompilationMode::kTurbofan: + new (&tf_) TurbofanData(); + break; + } +} + +WasmCompilationUnit::~WasmCompilationUnit() { + switch (mode_) { + case WasmCompilationUnit::CompilationMode::kLiftoff: + liftoff_.~LiftoffData(); + break; + case WasmCompilationUnit::CompilationMode::kTurbofan: + tf_.~TurbofanData(); + break; + } +} void WasmCompilationUnit::ExecuteCompilation() { + auto size_histogram = env_->module->is_wasm() + ? counters()->wasm_wasm_function_size_bytes() + : counters()->wasm_asm_function_size_bytes(); + size_histogram->AddSample( + static_cast<int>(func_body_.end - func_body_.start)); auto timed_histogram = env_->module->is_wasm() ? counters()->wasm_compile_wasm_function_time() : counters()->wasm_compile_asm_function_time(); @@ -4679,13 +4939,28 @@ void WasmCompilationUnit::ExecuteCompilation() { } } + switch (mode_) { + case WasmCompilationUnit::CompilationMode::kLiftoff: + if (ExecuteLiftoffCompilation()) break; + // Otherwise, fall back to turbofan. + liftoff_.~LiftoffData(); + mode_ = WasmCompilationUnit::CompilationMode::kTurbofan; + new (&tf_) TurbofanData(); + // fall-through + case WasmCompilationUnit::CompilationMode::kTurbofan: + ExecuteTurbofanCompilation(); + break; + } +} + +void WasmCompilationUnit::ExecuteTurbofanCompilation() { double decode_ms = 0; size_t node_count = 0; // Scope for the {graph_zone}. { Zone graph_zone(isolate_->allocator(), ZONE_NAME); - jsgraph_ = new (&graph_zone) JSGraph( + tf_.jsgraph_ = new (&graph_zone) JSGraph( isolate_, new (&graph_zone) Graph(&graph_zone), new (&graph_zone) CommonOperatorBuilder(&graph_zone), nullptr, nullptr, new (&graph_zone) MachineOperatorBuilder( @@ -4695,39 +4970,37 @@ void WasmCompilationUnit::ExecuteCompilation() { SourcePositionTable* source_positions = BuildGraphForWasmFunction(&decode_ms); - if (graph_construction_result_.failed()) { + if (tf_.graph_construction_result_.failed()) { ok_ = false; return; } base::ElapsedTimer pipeline_timer; if (FLAG_trace_wasm_decode_time) { - node_count = jsgraph_->graph()->NodeCount(); + node_count = tf_.jsgraph_->graph()->NodeCount(); pipeline_timer.Start(); } - compilation_zone_.reset(new Zone(isolate_->allocator(), ZONE_NAME)); + tf_.compilation_zone_.reset(new Zone(isolate_->allocator(), ZONE_NAME)); // Run the compiler pipeline to generate machine code. CallDescriptor* descriptor = - GetWasmCallDescriptor(compilation_zone_.get(), func_body_.sig); - if (jsgraph_->machine()->Is32()) { + GetWasmCallDescriptor(tf_.compilation_zone_.get(), func_body_.sig); + if (tf_.jsgraph_->machine()->Is32()) { descriptor = - GetI32WasmCallDescriptor(compilation_zone_.get(), descriptor); + GetI32WasmCallDescriptor(tf_.compilation_zone_.get(), descriptor); } - info_.reset(new CompilationInfo( - GetDebugName(compilation_zone_.get(), func_name_, func_index_), - isolate_, compilation_zone_.get(), Code::WASM_FUNCTION)); - ZoneVector<trap_handler::ProtectedInstructionData> protected_instructions( - compilation_zone_.get()); - - job_.reset(Pipeline::NewWasmCompilationJob( - info_.get(), jsgraph_, descriptor, source_positions, - &protected_instructions, env_->module->origin())); - ok_ = job_->ExecuteJob() == CompilationJob::SUCCEEDED; + tf_.info_.reset(new CompilationInfo( + GetDebugName(tf_.compilation_zone_.get(), func_name_, func_index_), + tf_.compilation_zone_.get(), Code::WASM_FUNCTION)); + + tf_.job_.reset(Pipeline::NewWasmCompilationJob( + tf_.info_.get(), isolate_, tf_.jsgraph_, descriptor, source_positions, + protected_instructions_.get(), env_->module->origin())); + ok_ = tf_.job_->ExecuteJob() == CompilationJob::SUCCEEDED; // TODO(bradnelson): Improve histogram handling of size_t. counters()->wasm_compile_function_peak_memory_bytes()->AddSample( - static_cast<int>(jsgraph_->graph()->zone()->allocation_size())); + static_cast<int>(tf_.jsgraph_->graph()->zone()->allocation_size())); if (FLAG_trace_wasm_decode_time) { double pipeline_ms = pipeline_timer.Elapsed().InMillisecondsF(); @@ -4738,19 +5011,40 @@ void WasmCompilationUnit::ExecuteCompilation() { node_count, pipeline_ms); } // The graph zone is about to get out of scope. Avoid invalid references. - jsgraph_ = nullptr; + tf_.jsgraph_ = nullptr; } // Record the memory cost this unit places on the system until // it is finalized. - size_t cost = job_->AllocatedMemory(); - set_memory_cost(cost); + memory_cost_ = tf_.job_->AllocatedMemory(); +} + +// WasmCompilationUnit::ExecuteLiftoffCompilation() is defined in +// liftoff-compiler.cc. + +WasmCodeWrapper WasmCompilationUnit::FinishCompilation( + wasm::ErrorThrower* thrower) { + WasmCodeWrapper ret; + switch (mode_) { + case WasmCompilationUnit::CompilationMode::kLiftoff: + ret = FinishLiftoffCompilation(thrower); + break; + case WasmCompilationUnit::CompilationMode::kTurbofan: + ret = FinishTurbofanCompilation(thrower); + break; + default: + UNREACHABLE(); + } + if (!ret.IsCodeObject() && ret.is_null()) { + thrower->RuntimeError("Error finalizing code."); + } + return ret; } -MaybeHandle<Code> WasmCompilationUnit::FinishCompilation( +WasmCodeWrapper WasmCompilationUnit::FinishTurbofanCompilation( wasm::ErrorThrower* thrower) { if (!ok_) { - if (graph_construction_result_.failed()) { + if (tf_.graph_construction_result_.failed()) { // Add the function as another context for the exception. EmbeddedVector<char, 128> message; if (func_name_.start() == nullptr) { @@ -4760,7 +5054,7 @@ MaybeHandle<Code> WasmCompilationUnit::FinishCompilation( SNPrintF(message, "Compiling wasm function #%d:%.*s failed", func_index_, trunc_name.length(), trunc_name.start()); } - thrower->CompileFailed(message.start(), graph_construction_result_); + thrower->CompileFailed(message.start(), tf_.graph_construction_result_); } return {}; @@ -4769,41 +5063,137 @@ MaybeHandle<Code> WasmCompilationUnit::FinishCompilation( if (FLAG_trace_wasm_decode_time) { codegen_timer.Start(); } - if (job_->FinalizeJob() != CompilationJob::SUCCEEDED) { - return Handle<Code>::null(); - } - Handle<Code> code = info_->code(); - DCHECK(!code.is_null()); - if (must_record_function_compilation(isolate_)) { - wasm::TruncatedUserString<> trunc_name(func_name_); - RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate_, code, - "wasm_function#%d:%.*s", func_index_, - trunc_name.length(), trunc_name.start()); + if (tf_.job_->FinalizeJob(isolate_) != CompilationJob::SUCCEEDED) { + return {}; } + if (!FLAG_wasm_jit_to_native) { + Handle<Code> code = tf_.info_->code(); + DCHECK(!code.is_null()); - if (FLAG_trace_wasm_decode_time) { - double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); - PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", - static_cast<unsigned>(func_body_.end - func_body_.start), - codegen_ms); - } + if (FLAG_trace_wasm_decode_time) { + double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); + PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", + static_cast<unsigned>(func_body_.end - func_body_.start), + codegen_ms); + } - return code; + PackProtectedInstructions(code); + return WasmCodeWrapper(code); + } else { + // TODO(mtrofin): when we crystalize a design in lieu of WasmCodeDesc, that + // works for both wasm and non-wasm, we can simplify AddCode to just take + // that as a parameter. + const CodeDesc& desc = + tf_.job_->compilation_info()->wasm_code_desc()->code_desc; + wasm::WasmCode* code = native_module_->AddCode( + desc, tf_.job_->compilation_info()->wasm_code_desc()->frame_slot_count, + func_index_, + tf_.job_->compilation_info()->wasm_code_desc()->safepoint_table_offset, + protected_instructions_); + if (!code) { + return WasmCodeWrapper(code); + } + // TODO(mtrofin): add CodeEventListener call - see the non-native case. + if (FLAG_trace_wasm_decode_time) { + double codegen_ms = codegen_timer.Elapsed().InMillisecondsF(); + PrintF("wasm-code-generation ok: %u bytes, %0.3f ms code generation\n", + static_cast<unsigned>(func_body_.end - func_body_.start), + codegen_ms); + } + + Handle<ByteArray> source_positions = + tf_.job_->compilation_info()->wasm_code_desc()->source_positions_table; + MaybeHandle<HandlerTable> handler_table = + tf_.job_->compilation_info()->wasm_code_desc()->handler_table; + + int function_index_as_int = static_cast<int>(func_index_); + native_module_->compiled_module()->source_positions()->set( + function_index_as_int, *source_positions); + if (!handler_table.is_null()) { + native_module_->compiled_module()->handler_table()->set( + function_index_as_int, *handler_table.ToHandleChecked()); + } + // TODO(mtrofin): this should probably move up in the common caller, + // once liftoff has source positions. Until then, we'd need to handle + // undefined values, which is complicating the code. + LOG_CODE_EVENT(isolate_, + CodeLinePosInfoRecordEvent(code->instructions().start(), + *source_positions)); + return WasmCodeWrapper(code); + } +} + +// TODO(mtrofin): remove when FLAG_wasm_jit_to_native is not needed +void WasmCompilationUnit::PackProtectedInstructions(Handle<Code> code) const { + if (protected_instructions_->empty()) return; + DCHECK_LT(protected_instructions_->size(), std::numeric_limits<int>::max()); + const int num_instructions = + static_cast<int>(protected_instructions_->size()); + Handle<FixedArray> fn_protected = isolate_->factory()->NewFixedArray( + num_instructions * Code::kTrapDataSize, TENURED); + for (int i = 0; i < num_instructions; ++i) { + const trap_handler::ProtectedInstructionData& instruction = + protected_instructions_->at(i); + fn_protected->set(Code::kTrapDataSize * i + Code::kTrapCodeOffset, + Smi::FromInt(instruction.instr_offset)); + fn_protected->set(Code::kTrapDataSize * i + Code::kTrapLandingOffset, + Smi::FromInt(instruction.landing_offset)); + } + code->set_protected_instructions(*fn_protected); +} + +WasmCodeWrapper WasmCompilationUnit::FinishLiftoffCompilation( + wasm::ErrorThrower* thrower) { + CodeDesc desc; + liftoff_.asm_.GetCode(isolate_, &desc); + WasmCodeWrapper ret; + if (!FLAG_wasm_jit_to_native) { + Handle<Code> code; + code = isolate_->factory()->NewCode(desc, Code::WASM_FUNCTION, code); +#ifdef ENABLE_DISASSEMBLER + if (FLAG_print_code || FLAG_print_wasm_code) { + // TODO(wasm): Use proper log files, here and elsewhere. + OFStream os(stdout); + os << "--- Wasm liftoff code ---\n"; + EmbeddedVector<char, 32> func_name; + func_name.Truncate(SNPrintF(func_name, "wasm#%d-liftoff", func_index_)); + code->Disassemble(func_name.start(), os); + os << "--- End code ---\n"; + } +#endif + if (isolate_->logger()->is_logging_code_events() || + isolate_->is_profiling()) { + RecordFunctionCompilation(CodeEventListener::FUNCTION_TAG, isolate_, code, + "wasm#%d-liftoff", func_index_); + } + + PackProtectedInstructions(code); + return WasmCodeWrapper(code); + } else { + // TODO(mtrofin): figure a way to raise events; also, disassembly. + // Consider lifting them both to FinishCompilation. + return WasmCodeWrapper(native_module_->AddCode( + desc, liftoff_.asm_.GetTotalFrameSlotCount(), func_index_, + liftoff_.asm_.GetSafepointTableOffset(), protected_instructions_, + true)); + } } // static -MaybeHandle<Code> WasmCompilationUnit::CompileWasmFunction( - wasm::ErrorThrower* thrower, Isolate* isolate, - const wasm::ModuleWireBytes& wire_bytes, ModuleEnv* env, - const wasm::WasmFunction* function) { +WasmCodeWrapper WasmCompilationUnit::CompileWasmFunction( + wasm::NativeModule* native_module, wasm::ErrorThrower* thrower, + Isolate* isolate, const wasm::ModuleWireBytes& wire_bytes, ModuleEnv* env, + const wasm::WasmFunction* function, CompilationMode mode) { wasm::FunctionBody function_body{ function->sig, function->code.offset(), wire_bytes.start() + function->code.offset(), wire_bytes.start() + function->code.end_offset()}; - WasmCompilationUnit unit( - isolate, env, function_body, wire_bytes.GetNameOrNull(function), - function->func_index, CEntryStub(isolate, 1).GetCode()); + + WasmCompilationUnit unit(isolate, env, native_module, function_body, + wire_bytes.GetNameOrNull(function), + function->func_index, + CEntryStub(isolate, 1).GetCode(), mode); unit.ExecuteCompilation(); return unit.FinishCompilation(thrower); } |