diff options
Diffstat (limited to 'deps/v8/src/wasm/module-compiler.cc')
| -rw-r--r-- | deps/v8/src/wasm/module-compiler.cc | 2138 |
1 files changed, 1438 insertions, 700 deletions
diff --git a/deps/v8/src/wasm/module-compiler.cc b/deps/v8/src/wasm/module-compiler.cc index e42c139ce1..4bd52a2a8f 100644 --- a/deps/v8/src/wasm/module-compiler.cc +++ b/deps/v8/src/wasm/module-compiler.cc @@ -9,15 +9,18 @@ #include "src/api.h" #include "src/asmjs/asm-js.h" #include "src/assembler-inl.h" +#include "src/base/optional.h" #include "src/base/template-utils.h" #include "src/base/utils/random-number-generator.h" #include "src/code-stubs.h" #include "src/compiler/wasm-compiler.h" #include "src/counters.h" #include "src/property-descriptor.h" +#include "src/trap-handler/trap-handler.h" #include "src/wasm/compilation-manager.h" #include "src/wasm/module-decoder.h" #include "src/wasm/wasm-code-specialization.h" +#include "src/wasm/wasm-heap.h" #include "src/wasm/wasm-js.h" #include "src/wasm/wasm-memory.h" #include "src/wasm/wasm-objects-inl.h" @@ -42,6 +45,12 @@ do { \ if (FLAG_trace_wasm_streaming) PrintF(__VA_ARGS__); \ } while (false) + +#define TRACE_LAZY(...) \ + do { \ + if (FLAG_trace_wasm_lazy_compilation) PrintF(__VA_ARGS__); \ + } while (false) + static const int kInvalidSigIndex = -1; namespace v8 { @@ -51,8 +60,8 @@ namespace wasm { // A class compiling an entire module. class ModuleCompiler { public: - ModuleCompiler(Isolate* isolate, WasmModule* module, - Handle<Code> centry_stub); + ModuleCompiler(Isolate* isolate, WasmModule* module, Handle<Code> centry_stub, + wasm::NativeModule* native_module); // The actual runnable task that performs compilations in the background. class CompilationTask : public CancelableTask { @@ -81,14 +90,16 @@ class ModuleCompiler { ~CompilationUnitBuilder() { DCHECK(units_.empty()); } - void AddUnit(compiler::ModuleEnv* module_env, const WasmFunction* function, - uint32_t buffer_offset, Vector<const uint8_t> bytes, - WasmName name) { + void AddUnit(compiler::ModuleEnv* module_env, + wasm::NativeModule* native_module, + const WasmFunction* function, uint32_t buffer_offset, + Vector<const uint8_t> bytes, WasmName name) { units_.emplace_back(new compiler::WasmCompilationUnit( - compiler_->isolate_, module_env, + compiler_->isolate_, module_env, native_module, wasm::FunctionBody{function->sig, buffer_offset, bytes.begin(), bytes.end()}, name, function->func_index, compiler_->centry_stub_, + compiler::WasmCompilationUnit::GetDefaultCompilationMode(), compiler_->counters())); } @@ -171,8 +182,7 @@ class ModuleCompiler { void SetFinisherIsRunning(bool value); - MaybeHandle<Code> FinishCompilationUnit(ErrorThrower* thrower, - int* func_index); + WasmCodeWrapper FinishCompilationUnit(ErrorThrower* thrower, int* func_index); void CompileInParallel(const ModuleWireBytes& wire_bytes, compiler::ModuleEnv* module_env, @@ -215,8 +225,11 @@ class ModuleCompiler { size_t stopped_compilation_tasks_ = 0; base::Mutex tasks_mutex_; Handle<Code> centry_stub_; + wasm::NativeModule* native_module_; }; +namespace { + class JSToWasmWrapperCache { public: void SetContextAddress(Address context_address) { @@ -227,23 +240,37 @@ class JSToWasmWrapperCache { Handle<Code> CloneOrCompileJSToWasmWrapper(Isolate* isolate, wasm::WasmModule* module, - Handle<Code> wasm_code, + WasmCodeWrapper wasm_code, uint32_t index) { const wasm::WasmFunction* func = &module->functions[index]; int cached_idx = sig_map_.Find(func->sig); if (cached_idx >= 0) { Handle<Code> code = isolate->factory()->CopyCode(code_cache_[cached_idx]); // Now patch the call to wasm code. - for (RelocIterator it(*code, RelocInfo::kCodeTargetMask);; it.next()) { - DCHECK(!it.done()); - Code* target = - Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); - if (target->kind() == Code::WASM_FUNCTION || - target->kind() == Code::WASM_TO_JS_FUNCTION || - target->builtin_index() == Builtins::kIllegal || - target->builtin_index() == Builtins::kWasmCompileLazy) { - it.rinfo()->set_target_address(isolate, - wasm_code->instruction_start()); + if (wasm_code.IsCodeObject()) { + for (RelocIterator it(*code, RelocInfo::kCodeTargetMask);; it.next()) { + DCHECK(!it.done()); + Code* target = + Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); + if (target->kind() == Code::WASM_FUNCTION || + target->kind() == Code::WASM_TO_JS_FUNCTION || + target->kind() == Code::WASM_TO_WASM_FUNCTION || + target->builtin_index() == Builtins::kIllegal || + target->builtin_index() == Builtins::kWasmCompileLazy) { + it.rinfo()->set_target_address( + isolate, wasm_code.GetCode()->instruction_start()); + break; + } + } + } else { + for (RelocIterator it(*code, + RelocInfo::ModeMask(RelocInfo::JS_TO_WASM_CALL)); + ; it.next()) { + DCHECK(!it.done()); + it.rinfo()->set_js_to_wasm_address( + isolate, wasm_code.is_null() + ? nullptr + : wasm_code.GetWasmCode()->instructions().start()); break; } } @@ -335,6 +362,8 @@ class InstanceBuilder { ERROR_THROWER_WITH_MESSAGE(LinkError) ERROR_THROWER_WITH_MESSAGE(TypeError) +#undef ERROR_THROWER_WITH_MESSAGE + // Look up an import value in the {ffi_} object. MaybeHandle<Object> LookupImport(uint32_t index, Handle<String> module_name, Handle<String> import_name); @@ -349,7 +378,7 @@ class InstanceBuilder { uint32_t EvalUint32InitExpr(const WasmInitExpr& expr); // Load data segments into the memory. - void LoadDataSegments(Address mem_addr, size_t mem_size); + void LoadDataSegments(WasmContext* wasm_context); void WriteGlobalValue(WasmGlobal& global, Handle<Object> value); @@ -388,7 +417,7 @@ class InstanceBuilder { }; // TODO(titzer): move to wasm-objects.cc -static void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) { +void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) { DisallowHeapAllocation no_gc; JSObject** p = reinterpret_cast<JSObject**>(data.GetParameter()); WasmInstanceObject* owner = reinterpret_cast<WasmInstanceObject*>(*p); @@ -396,21 +425,32 @@ static void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) { // If a link to shared memory instances exists, update the list of memory // instances before the instance is destroyed. WasmCompiledModule* compiled_module = owner->compiled_module(); - TRACE("Finalizing %d {\n", compiled_module->instance_id()); - DCHECK(compiled_module->has_weak_wasm_module()); - WeakCell* weak_wasm_module = compiled_module->ptr_to_weak_wasm_module(); + wasm::NativeModule* native_module = compiled_module->GetNativeModule(); + if (FLAG_wasm_jit_to_native) { + if (native_module) { + TRACE("Finalizing %zu {\n", native_module->instance_id); + } else { + TRACE("Finalized already cleaned up compiled module\n"); + } + } else { + TRACE("Finalizing %d {\n", compiled_module->instance_id()); - if (trap_handler::UseTrapHandler()) { - Handle<FixedArray> code_table = compiled_module->code_table(); - for (int i = 0; i < code_table->length(); ++i) { - Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i); - int index = code->trap_handler_index()->value(); - if (index >= 0) { - trap_handler::ReleaseHandlerData(index); - code->set_trap_handler_index(Smi::FromInt(trap_handler::kInvalidIndex)); + if (trap_handler::UseTrapHandler()) { + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(isolate->heap()); + Handle<FixedArray> code_table = compiled_module->code_table(); + for (int i = 0; i < code_table->length(); ++i) { + Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i); + int index = code->trap_handler_index()->value(); + if (index >= 0) { + trap_handler::ReleaseHandlerData(index); + code->set_trap_handler_index( + Smi::FromInt(trap_handler::kInvalidIndex)); + } } } } + WeakCell* weak_wasm_module = compiled_module->ptr_to_weak_wasm_module(); // Since the order of finalizers is not guaranteed, it can be the case // that {instance->compiled_module()->module()}, which is a @@ -425,56 +465,32 @@ static void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) { } // weak_wasm_module may have been cleared, meaning the module object - // was GC-ed. In that case, there won't be any new instances created, - // and we don't need to maintain the links between instances. + // was GC-ed. We still want to maintain the links between instances, to + // release the WasmCompiledModule corresponding to the WasmModuleInstance + // being finalized here. + WasmModuleObject* wasm_module = nullptr; if (!weak_wasm_module->cleared()) { - WasmModuleObject* wasm_module = - WasmModuleObject::cast(weak_wasm_module->value()); + wasm_module = WasmModuleObject::cast(weak_wasm_module->value()); WasmCompiledModule* current_template = wasm_module->compiled_module(); TRACE("chain before {\n"); TRACE_CHAIN(current_template); TRACE("}\n"); - DCHECK(!current_template->has_weak_prev_instance()); - WeakCell* next = compiled_module->maybe_ptr_to_weak_next_instance(); - WeakCell* prev = compiled_module->maybe_ptr_to_weak_prev_instance(); - + DCHECK(!current_template->has_prev_instance()); if (current_template == compiled_module) { - if (next == nullptr) { + if (!compiled_module->has_next_instance()) { WasmCompiledModule::Reset(isolate, compiled_module); } else { - WasmCompiledModule* next_compiled_module = - WasmCompiledModule::cast(next->value()); WasmModuleObject::cast(wasm_module) - ->set_compiled_module(next_compiled_module); - DCHECK_NULL(prev); - next_compiled_module->reset_weak_prev_instance(); - } - } else { - DCHECK(!(prev == nullptr && next == nullptr)); - // the only reason prev or next would be cleared is if the - // respective objects got collected, but if that happened, - // we would have relinked the list. - if (prev != nullptr) { - DCHECK(!prev->cleared()); - if (next == nullptr) { - WasmCompiledModule::cast(prev->value())->reset_weak_next_instance(); - } else { - WasmCompiledModule::cast(prev->value()) - ->set_ptr_to_weak_next_instance(next); - } - } - if (next != nullptr) { - DCHECK(!next->cleared()); - if (prev == nullptr) { - WasmCompiledModule::cast(next->value())->reset_weak_prev_instance(); - } else { - WasmCompiledModule::cast(next->value()) - ->set_ptr_to_weak_prev_instance(prev); - } + ->set_compiled_module(compiled_module->ptr_to_next_instance()); } } + } + + compiled_module->RemoveFromChain(); + + if (wasm_module != nullptr) { TRACE("chain after {\n"); TRACE_CHAIN(wasm_module->compiled_module()); TRACE("}\n"); @@ -484,6 +500,8 @@ static void InstanceFinalizer(const v8::WeakCallbackInfo<void>& data) { TRACE("}\n"); } +} // namespace + bool SyncValidate(Isolate* isolate, const ModuleWireBytes& bytes) { if (bytes.start() == nullptr || bytes.length() == 0) return false; ModuleResult result = SyncDecodeWasmModule(isolate, bytes.start(), @@ -547,9 +565,8 @@ MaybeHandle<WasmInstanceObject> SyncCompileAndInstantiate( DCHECK_EQ(thrower->error(), module.is_null()); if (module.is_null()) return {}; - return SyncInstantiate(isolate, thrower, module.ToHandleChecked(), - Handle<JSReceiver>::null(), - Handle<JSArrayBuffer>::null()); + return SyncInstantiate(isolate, thrower, module.ToHandleChecked(), imports, + memory); } void RejectPromise(Isolate* isolate, Handle<Context> context, @@ -617,7 +634,7 @@ void AsyncCompile(Isolate* isolate, Handle<JSPromise> promise, promise); } -Handle<Code> CompileLazy(Isolate* isolate) { +Handle<Code> CompileLazyOnGCHeap(Isolate* isolate) { HistogramTimerScope lazy_time_scope( isolate->counters()->wasm_lazy_compilation_time()); @@ -635,13 +652,14 @@ Handle<Code> CompileLazy(Isolate* isolate) { Handle<WasmInstanceObject> instance; Handle<FixedArray> exp_deopt_data; int func_index = -1; + // If the lazy compile stub has deopt data, use that to determine the + // instance and function index. Otherwise this must be a wasm->wasm call + // within one instance, so extract the information from the caller. if (lazy_compile_code->deoptimization_data()->length() > 0) { - // Then it's an indirect call or via JS->wasm wrapper. - DCHECK_LE(2, lazy_compile_code->deoptimization_data()->length()); exp_deopt_data = handle(lazy_compile_code->deoptimization_data(), isolate); - auto* weak_cell = WeakCell::cast(exp_deopt_data->get(0)); - instance = handle(WasmInstanceObject::cast(weak_cell->value()), isolate); - func_index = Smi::ToInt(exp_deopt_data->get(1)); + auto func_info = GetWasmFunctionInfo(isolate, lazy_compile_code); + instance = func_info.instance.ToHandleChecked(); + func_index = func_info.func_index; } it.Advance(); // Third frame: The calling wasm code or js-to-wasm wrapper. @@ -655,7 +673,7 @@ Handle<Code> CompileLazy(Isolate* isolate) { // via deopt data to the lazy compile stub). Just use the instance of the // caller. instance = - handle(WasmInstanceObject::GetOwningInstance(*caller_code), isolate); + handle(WasmInstanceObject::GetOwningInstanceGC(*caller_code), isolate); } int offset = static_cast<int>(it.frame()->pc() - caller_code->instruction_start()); @@ -668,9 +686,17 @@ Handle<Code> CompileLazy(Isolate* isolate) { bool patch_caller = caller_code->kind() == Code::JS_TO_WASM_FUNCTION || exp_deopt_data.is_null() || exp_deopt_data->length() <= 2; - Handle<Code> compiled_code = WasmCompiledModule::CompileLazy( + wasm::LazyCompilationOrchestrator* orchestrator = + Managed<wasm::LazyCompilationOrchestrator>::cast( + instance->compiled_module() + ->shared() + ->lazy_compilation_orchestrator()) + ->get(); + Handle<Code> compiled_code = orchestrator->CompileLazyOnGCHeap( isolate, instance, caller_code, offset, func_index, patch_caller); if (!exp_deopt_data.is_null() && exp_deopt_data->length() > 2) { + TRACE_LAZY("Patching %d position(s) in function tables.\n", + (exp_deopt_data->length() - 2) / 2); // See EnsureExportedLazyDeoptData: exp_deopt_data[2...(len-1)] are pairs of // <export_table, index> followed by undefined values. // Use this information here to patch all export tables. @@ -682,6 +708,8 @@ Handle<Code> CompileLazy(Isolate* isolate) { DCHECK(exp_table->get(exp_index) == *lazy_compile_code); exp_table->set(exp_index, *compiled_code); } + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(isolate->heap()); // After processing, remove the list of exported entries, such that we don't // do the patching redundantly. Handle<FixedArray> new_deopt_data = @@ -692,48 +720,178 @@ Handle<Code> CompileLazy(Isolate* isolate) { return compiled_code; } +Address CompileLazy(Isolate* isolate) { + HistogramTimerScope lazy_time_scope( + isolate->counters()->wasm_lazy_compilation_time()); + + // Find the wasm frame which triggered the lazy compile, to get the wasm + // instance. + StackFrameIterator it(isolate); + // First frame: C entry stub. + DCHECK(!it.done()); + DCHECK_EQ(StackFrame::EXIT, it.frame()->type()); + it.Advance(); + // Second frame: WasmCompileLazy builtin. + DCHECK(!it.done()); + Handle<WasmInstanceObject> instance; + Maybe<uint32_t> func_index_to_compile = Nothing<uint32_t>(); + Handle<Object> exp_deopt_data_entry; + const wasm::WasmCode* lazy_stub_or_copy = + isolate->wasm_code_manager()->LookupCode(it.frame()->pc()); + DCHECK_EQ(wasm::WasmCode::LazyStub, lazy_stub_or_copy->kind()); + if (!lazy_stub_or_copy->IsAnonymous()) { + // Then it's an indirect call or via JS->wasm wrapper. + instance = lazy_stub_or_copy->owner()->compiled_module()->owning_instance(); + func_index_to_compile = Just(lazy_stub_or_copy->index()); + exp_deopt_data_entry = + handle(instance->compiled_module()->lazy_compile_data()->get( + static_cast<int>(lazy_stub_or_copy->index())), + isolate); + } + it.Advance(); + // Third frame: The calling wasm code (direct or indirect), or js-to-wasm + // wrapper. + DCHECK(!it.done()); + DCHECK(it.frame()->is_js_to_wasm() || it.frame()->is_wasm_compiled()); + Handle<Code> js_to_wasm_caller_code; + const WasmCode* wasm_caller_code = nullptr; + Maybe<uint32_t> offset = Nothing<uint32_t>(); + if (it.frame()->is_js_to_wasm()) { + DCHECK(!instance.is_null()); + js_to_wasm_caller_code = handle(it.frame()->LookupCode(), isolate); + } else { + wasm_caller_code = + isolate->wasm_code_manager()->LookupCode(it.frame()->pc()); + offset = Just(static_cast<uint32_t>( + it.frame()->pc() - wasm_caller_code->instructions().start())); + if (instance.is_null()) { + // Then this is a direct call (otherwise we would have attached the + // instance via deopt data to the lazy compile stub). Just use the + // instance of the caller. + instance = + wasm_caller_code->owner()->compiled_module()->owning_instance(); + } + } + + Handle<WasmCompiledModule> compiled_module(instance->compiled_module()); + + wasm::LazyCompilationOrchestrator* orchestrator = + Managed<wasm::LazyCompilationOrchestrator>::cast( + compiled_module->shared()->lazy_compilation_orchestrator()) + ->get(); + const wasm::WasmCode* result = nullptr; + // The caller may be js to wasm calling a function + // also available for indirect calls. + if (!js_to_wasm_caller_code.is_null()) { + result = orchestrator->CompileFromJsToWasm( + isolate, instance, js_to_wasm_caller_code, + func_index_to_compile.ToChecked()); + } else { + DCHECK_NOT_NULL(wasm_caller_code); + if (func_index_to_compile.IsNothing() || + (!exp_deopt_data_entry.is_null() && + !exp_deopt_data_entry->IsFixedArray())) { + result = orchestrator->CompileDirectCall( + isolate, instance, func_index_to_compile, wasm_caller_code, + offset.ToChecked()); + } else { + result = orchestrator->CompileIndirectCall( + isolate, instance, func_index_to_compile.ToChecked()); + } + } + DCHECK_NOT_NULL(result); + + int func_index = static_cast<int>(result->index()); + if (!exp_deopt_data_entry.is_null() && exp_deopt_data_entry->IsFixedArray()) { + Handle<FixedArray> exp_deopt_data = + Handle<FixedArray>::cast(exp_deopt_data_entry); + + TRACE_LAZY("Patching %d position(s) in function tables.\n", + exp_deopt_data->length() / 2); + + // See EnsureExportedLazyDeoptData: exp_deopt_data[0...(len-1)] are pairs + // of <export_table, index> followed by undefined values. Use this + // information here to patch all export tables. + for (int idx = 0, end = exp_deopt_data->length(); idx < end; idx += 2) { + if (exp_deopt_data->get(idx)->IsUndefined(isolate)) break; + FixedArray* exp_table = FixedArray::cast(exp_deopt_data->get(idx)); + int exp_index = Smi::ToInt(exp_deopt_data->get(idx + 1)); + Handle<Foreign> foreign_holder = isolate->factory()->NewForeign( + result->instructions().start(), TENURED); + exp_table->set(exp_index, *foreign_holder); + } + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(isolate->heap()); + // After processing, remove the list of exported entries, such that we don't + // do the patching redundantly. + compiled_module->lazy_compile_data()->set( + func_index, isolate->heap()->undefined_value()); + } + + return result->instructions().start(); +} + compiler::ModuleEnv CreateModuleEnvFromCompiledModule( Isolate* isolate, Handle<WasmCompiledModule> compiled_module) { DisallowHeapAllocation no_gc; WasmModule* module = compiled_module->module(); + std::vector<Handle<Code>> empty_code; + if (FLAG_wasm_jit_to_native) { + NativeModule* native_module = compiled_module->GetNativeModule(); + std::vector<GlobalHandleAddress> function_tables = + native_module->function_tables(); + std::vector<GlobalHandleAddress> signature_tables = + native_module->signature_tables(); + + compiler::ModuleEnv result = {module, // -- + function_tables, // -- + signature_tables, // -- + empty_code, + BUILTIN_CODE(isolate, WasmCompileLazy)}; + return result; + } else { + std::vector<GlobalHandleAddress> function_tables; + std::vector<GlobalHandleAddress> signature_tables; - std::vector<GlobalHandleAddress> function_tables; - std::vector<GlobalHandleAddress> signature_tables; - std::vector<SignatureMap*> signature_maps; + int num_function_tables = static_cast<int>(module->function_tables.size()); + for (int i = 0; i < num_function_tables; ++i) { + FixedArray* ft = compiled_module->ptr_to_function_tables(); + FixedArray* st = compiled_module->ptr_to_signature_tables(); - int num_function_tables = static_cast<int>(module->function_tables.size()); - for (int i = 0; i < num_function_tables; ++i) { - FixedArray* ft = compiled_module->ptr_to_function_tables(); - FixedArray* st = compiled_module->ptr_to_signature_tables(); + // TODO(clemensh): defer these handles for concurrent compilation. + function_tables.push_back(WasmCompiledModule::GetTableValue(ft, i)); + signature_tables.push_back(WasmCompiledModule::GetTableValue(st, i)); + } - // TODO(clemensh): defer these handles for concurrent compilation. - function_tables.push_back(WasmCompiledModule::GetTableValue(ft, i)); - signature_tables.push_back(WasmCompiledModule::GetTableValue(st, i)); - signature_maps.push_back(&module->function_tables[i].map); + compiler::ModuleEnv result = {module, // -- + function_tables, // -- + signature_tables, // -- + empty_code, // -- + BUILTIN_CODE(isolate, WasmCompileLazy)}; + return result; } - - std::vector<Handle<Code>> empty_code; - - compiler::ModuleEnv result = { - module, // -- - function_tables, // -- - signature_tables, // -- - signature_maps, // -- - empty_code, // -- - BUILTIN_CODE(isolate, WasmCompileLazy), // -- - reinterpret_cast<uintptr_t>( // -- - compiled_module->GetGlobalsStartOrNull()) // -- - }; - return result; } -void LazyCompilationOrchestrator::CompileFunction( +const wasm::WasmCode* LazyCompilationOrchestrator::CompileFunction( Isolate* isolate, Handle<WasmInstanceObject> instance, int func_index) { + base::ElapsedTimer compilation_timer; + compilation_timer.Start(); Handle<WasmCompiledModule> compiled_module(instance->compiled_module(), isolate); - if (Code::cast(compiled_module->code_table()->get(func_index))->kind() == - Code::WASM_FUNCTION) { - return; + if (FLAG_wasm_jit_to_native) { + wasm::WasmCode* existing_code = compiled_module->GetNativeModule()->GetCode( + static_cast<uint32_t>(func_index)); + if (existing_code != nullptr && + existing_code->kind() == wasm::WasmCode::Function) { + TRACE_LAZY("Function %d already compiled.\n", func_index); + return existing_code; + } + } else { + if (Code::cast(compiled_module->code_table()->get(func_index))->kind() == + Code::WASM_FUNCTION) { + TRACE_LAZY("Function %d already compiled.\n", func_index); + return nullptr; + } } compiler::ModuleEnv module_env = @@ -756,11 +914,14 @@ void LazyCompilationOrchestrator::CompileFunction( func_name.assign(name.start(), static_cast<size_t>(name.length())); } ErrorThrower thrower(isolate, "WasmLazyCompile"); - compiler::WasmCompilationUnit unit(isolate, &module_env, body, + compiler::WasmCompilationUnit unit(isolate, &module_env, + compiled_module->GetNativeModule(), body, CStrVector(func_name.c_str()), func_index, CEntryStub(isolate, 1).GetCode()); unit.ExecuteCompilation(); - MaybeHandle<Code> maybe_code = unit.FinishCompilation(&thrower); + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(isolate->heap()); + WasmCodeWrapper code_wrapper = unit.FinishCompilation(&thrower); // If there is a pending error, something really went wrong. The module was // verified before starting execution with lazy compilation. @@ -768,33 +929,53 @@ void LazyCompilationOrchestrator::CompileFunction( // TODO(clemensh): According to the spec, we can actually skip validation at // module creation time, and return a function that always traps here. CHECK(!thrower.error()); - Handle<Code> code = maybe_code.ToHandleChecked(); - - Handle<FixedArray> deopt_data = isolate->factory()->NewFixedArray(2, TENURED); - Handle<WeakCell> weak_instance = isolate->factory()->NewWeakCell(instance); - // TODO(wasm): Introduce constants for the indexes in wasm deopt data. - deopt_data->set(0, *weak_instance); - deopt_data->set(1, Smi::FromInt(func_index)); - code->set_deoptimization_data(*deopt_data); - - DCHECK_EQ(Builtins::kWasmCompileLazy, - Code::cast(compiled_module->code_table()->get(func_index)) - ->builtin_index()); - compiled_module->code_table()->set(func_index, *code); - // Now specialize the generated code for this instance. + + // {code} is used only when !FLAG_wasm_jit_to_native, so it may be removed + // when that flag is removed. + Handle<Code> code; + if (code_wrapper.IsCodeObject()) { + code = code_wrapper.GetCode(); + AttachWasmFunctionInfo(isolate, code, instance, func_index); + DCHECK_EQ(Builtins::kWasmCompileLazy, + Code::cast(compiled_module->code_table()->get(func_index)) + ->builtin_index()); + compiled_module->code_table()->set(func_index, *code); + } Zone specialization_zone(isolate->allocator(), ZONE_NAME); CodeSpecialization code_specialization(isolate, &specialization_zone); code_specialization.RelocateDirectCalls(instance); - code_specialization.ApplyToWasmCode(*code, SKIP_ICACHE_FLUSH); - Assembler::FlushICache(isolate, code->instruction_start(), - code->instruction_size()); + code_specialization.ApplyToWasmCode(code_wrapper, SKIP_ICACHE_FLUSH); + int64_t func_size = + static_cast<int64_t>(func->code.end_offset() - func->code.offset()); + int64_t compilation_time = compilation_timer.Elapsed().InMicroseconds(); + auto counters = isolate->counters(); counters->wasm_lazily_compiled_functions()->Increment(); - counters->wasm_generated_code_size()->Increment(code->body_size()); - counters->wasm_reloc_size()->Increment(code->relocation_info()->length()); + + if (!code_wrapper.IsCodeObject()) { + const wasm::WasmCode* wasm_code = code_wrapper.GetWasmCode(); + Assembler::FlushICache(isolate, wasm_code->instructions().start(), + wasm_code->instructions().size()); + counters->wasm_generated_code_size()->Increment( + static_cast<int>(wasm_code->instructions().size())); + counters->wasm_reloc_size()->Increment( + static_cast<int>(wasm_code->reloc_info().size())); + + } else { + Assembler::FlushICache(isolate, code->instruction_start(), + code->instruction_size()); + counters->wasm_generated_code_size()->Increment(code->body_size()); + counters->wasm_reloc_size()->Increment(code->relocation_info()->length()); + } + counters->wasm_lazy_compilation_throughput()->AddSample( + compilation_time != 0 ? static_cast<int>(func_size / compilation_time) + : 0); + return !code_wrapper.IsCodeObject() ? code_wrapper.GetWasmCode() : nullptr; } +namespace { + int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator, int offset) { DCHECK(!iterator.done()); @@ -806,7 +987,38 @@ int AdvanceSourcePositionTableIterator(SourcePositionTableIterator& iterator, return byte_pos; } -Handle<Code> LazyCompilationOrchestrator::CompileLazy( +Code* ExtractWasmToWasmCallee(Code* wasm_to_wasm) { + DCHECK_EQ(Code::WASM_TO_WASM_FUNCTION, wasm_to_wasm->kind()); + // Find the one code target in this wrapper. + RelocIterator it(wasm_to_wasm, RelocInfo::kCodeTargetMask); + DCHECK(!it.done()); + Code* callee = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); +#ifdef DEBUG + it.next(); + DCHECK(it.done()); +#endif + return callee; +} + +void PatchWasmToWasmWrapper(Isolate* isolate, Code* wasm_to_wasm, + Code* new_target) { + DCHECK_EQ(Code::WASM_TO_WASM_FUNCTION, wasm_to_wasm->kind()); + // Find the one code target in this wrapper. + RelocIterator it(wasm_to_wasm, RelocInfo::kCodeTargetMask); + DCHECK(!it.done()); + DCHECK_EQ(Builtins::kWasmCompileLazy, + Code::GetCodeFromTargetAddress(it.rinfo()->target_address()) + ->builtin_index()); + it.rinfo()->set_target_address(isolate, new_target->instruction_start()); +#ifdef DEBUG + it.next(); + DCHECK(it.done()); +#endif +} + +} // namespace + +Handle<Code> LazyCompilationOrchestrator::CompileLazyOnGCHeap( Isolate* isolate, Handle<WasmInstanceObject> instance, Handle<Code> caller, int call_offset, int exported_func_index, bool patch_caller) { struct NonCompiledFunction { @@ -820,65 +1032,122 @@ Handle<Code> LazyCompilationOrchestrator::CompileLazy( Handle<WasmCompiledModule> compiled_module(instance->compiled_module(), isolate); - if (is_js_to_wasm) { - non_compiled_functions.push_back({0, exported_func_index}); - } else if (patch_caller) { + TRACE_LAZY( + "Starting lazy compilation (func %d @%d, js-to-wasm: %d, " + "patch caller: %d).\n", + exported_func_index, call_offset, is_js_to_wasm, patch_caller); + + // If this lazy compile stub is being called through a wasm-to-wasm wrapper, + // remember that code object. + Handle<Code> wasm_to_wasm_callee; + + // For js-to-wasm wrappers, don't iterate the reloc info. There is just one + // call site in there anyway. + if (patch_caller && !is_js_to_wasm) { DisallowHeapAllocation no_gc; - SeqOneByteString* module_bytes = compiled_module->module_bytes(); SourcePositionTableIterator source_pos_iterator( caller->SourcePositionTable()); - DCHECK_EQ(2, caller->deoptimization_data()->length()); - int caller_func_index = Smi::ToInt(caller->deoptimization_data()->get(1)); + auto caller_func_info = GetWasmFunctionInfo(isolate, caller); + Handle<WasmCompiledModule> caller_module( + caller_func_info.instance.ToHandleChecked()->compiled_module(), + isolate); + SeqOneByteString* module_bytes = caller_module->module_bytes(); const byte* func_bytes = - module_bytes->GetChars() + - compiled_module->module()->functions[caller_func_index].code.offset(); + module_bytes->GetChars() + caller_module->module() + ->functions[caller_func_info.func_index] + .code.offset(); + Code* lazy_callee = nullptr; for (RelocIterator it(*caller, RelocInfo::kCodeTargetMask); !it.done(); it.next()) { Code* callee = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); - if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue; // TODO(clemensh): Introduce safe_cast<T, bool> which (D)CHECKS // (depending on the bool) against limits of T and then static_casts. size_t offset_l = it.rinfo()->pc() - caller->instruction_start(); DCHECK_GE(kMaxInt, offset_l); int offset = static_cast<int>(offset_l); + // Call offset points to the instruction after the call. Remember the last + // called code object before that offset. + if (offset < call_offset) lazy_callee = callee; + if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue; int byte_pos = AdvanceSourcePositionTableIterator(source_pos_iterator, offset); int called_func_index = ExtractDirectCallIndex(decoder, func_bytes + byte_pos); non_compiled_functions.push_back({offset, called_func_index}); - // Call offset one instruction after the call. Remember the last called - // function before that offset. if (offset < call_offset) func_to_return_idx = called_func_index; } + TRACE_LAZY("Found %zu non-compiled functions in caller.\n", + non_compiled_functions.size()); + DCHECK_NOT_NULL(lazy_callee); + if (lazy_callee->kind() == Code::WASM_TO_WASM_FUNCTION) { + TRACE_LAZY("Callee is a wasm-to-wasm.\n"); + wasm_to_wasm_callee = handle(lazy_callee, isolate); + // If we call a wasm-to-wasm wrapper, then this wrapper actually + // tail-called the lazy compile stub. Find it in the wrapper. + lazy_callee = ExtractWasmToWasmCallee(lazy_callee); + // This lazy compile stub belongs to the instance that was passed. + DCHECK_EQ(*instance, + *GetWasmFunctionInfo(isolate, handle(lazy_callee, isolate)) + .instance.ToHandleChecked()); + DCHECK_LE(2, lazy_callee->deoptimization_data()->length()); + func_to_return_idx = + Smi::ToInt(lazy_callee->deoptimization_data()->get(1)); + } + DCHECK_EQ(Builtins::kWasmCompileLazy, lazy_callee->builtin_index()); + // There must be at least one call to patch (the one that lead to calling + // the lazy compile stub). + DCHECK(!non_compiled_functions.empty() || !wasm_to_wasm_callee.is_null()); } + TRACE_LAZY("Compiling function %d.\n", func_to_return_idx); + // TODO(clemensh): compile all functions in non_compiled_functions in // background, wait for func_to_return_idx. CompileFunction(isolate, instance, func_to_return_idx); - if (is_js_to_wasm || patch_caller) { + Handle<Code> compiled_function( + Code::cast(compiled_module->code_table()->get(func_to_return_idx)), + isolate); + DCHECK_EQ(Code::WASM_FUNCTION, compiled_function->kind()); + + if (patch_caller || is_js_to_wasm) { DisallowHeapAllocation no_gc; + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(isolate->heap()); // Now patch the code object with all functions which are now compiled. int idx = 0; + int patched = 0; for (RelocIterator it(*caller, RelocInfo::kCodeTargetMask); !it.done(); it.next()) { Code* callee = Code::GetCodeFromTargetAddress(it.rinfo()->target_address()); - if (callee->builtin_index() != Builtins::kWasmCompileLazy) continue; - DCHECK_GT(non_compiled_functions.size(), idx); - int called_func_index = non_compiled_functions[idx].func_index; + if (callee->builtin_index() != Builtins::kWasmCompileLazy) { + // If the callee is the wasm-to-wasm wrapper triggering this lazy + // compilation, patch it. If is_js_to_wasm is set, we did not set the + // wasm_to_wasm_callee, so just check the code kind (this is the only + // call in that wrapper anyway). + if ((is_js_to_wasm && callee->kind() == Code::WASM_TO_WASM_FUNCTION) || + (!wasm_to_wasm_callee.is_null() && + callee == *wasm_to_wasm_callee)) { + TRACE_LAZY("Patching wasm-to-wasm wrapper.\n"); + PatchWasmToWasmWrapper(isolate, callee, *compiled_function); + ++patched; + } + continue; + } + int called_func_index = func_to_return_idx; + if (!is_js_to_wasm) { + DCHECK_GT(non_compiled_functions.size(), idx); + called_func_index = non_compiled_functions[idx].func_index; + DCHECK_EQ(non_compiled_functions[idx].offset, + it.rinfo()->pc() - caller->instruction_start()); + ++idx; + } // Check that the callee agrees with our assumed called_func_index. DCHECK_IMPLIES(callee->deoptimization_data()->length() > 0, Smi::ToInt(callee->deoptimization_data()->get(1)) == called_func_index); - if (is_js_to_wasm) { - DCHECK_EQ(func_to_return_idx, called_func_index); - } else { - DCHECK_EQ(non_compiled_functions[idx].offset, - it.rinfo()->pc() - caller->instruction_start()); - } - ++idx; Handle<Code> callee_compiled( Code::cast(compiled_module->code_table()->get(called_func_index))); if (callee_compiled->builtin_index() == Builtins::kWasmCompileLazy) { @@ -888,14 +1157,160 @@ Handle<Code> LazyCompilationOrchestrator::CompileLazy( DCHECK_EQ(Code::WASM_FUNCTION, callee_compiled->kind()); it.rinfo()->set_target_address(isolate, callee_compiled->instruction_start()); + ++patched; } DCHECK_EQ(non_compiled_functions.size(), idx); + TRACE_LAZY("Patched %d location(s) in the caller.\n", patched); + DCHECK_LT(0, patched); + USE(patched); + } + + return compiled_function; +} + +const wasm::WasmCode* LazyCompilationOrchestrator::CompileFromJsToWasm( + Isolate* isolate, Handle<WasmInstanceObject> instance, + Handle<Code> js_to_wasm_caller, uint32_t exported_func_index) { + Decoder decoder(nullptr, nullptr); + Handle<WasmCompiledModule> compiled_module(instance->compiled_module(), + isolate); + + TRACE_LAZY( + "Starting lazy compilation (func %u, js_to_wasm: true, patch caller: " + "true). \n", + exported_func_index); + CompileFunction(isolate, instance, exported_func_index); + { + DisallowHeapAllocation no_gc; + int idx = 0; + for (RelocIterator it(*js_to_wasm_caller, + RelocInfo::ModeMask(RelocInfo::JS_TO_WASM_CALL)); + !it.done(); it.next()) { + ++idx; + const wasm::WasmCode* callee_compiled = + compiled_module->GetNativeModule()->GetCode(exported_func_index); + DCHECK_NOT_NULL(callee_compiled); + it.rinfo()->set_js_to_wasm_address( + isolate, callee_compiled->instructions().start()); + } + DCHECK_EQ(1, idx); + } + + wasm::WasmCode* ret = + compiled_module->GetNativeModule()->GetCode(exported_func_index); + DCHECK_NOT_NULL(ret); + DCHECK_EQ(wasm::WasmCode::Function, ret->kind()); + return ret; +} + +const wasm::WasmCode* LazyCompilationOrchestrator::CompileIndirectCall( + Isolate* isolate, Handle<WasmInstanceObject> instance, + uint32_t func_index) { + TRACE_LAZY( + "Starting lazy compilation (func %u, js_to_wasm: false, patch caller: " + "false). \n", + func_index); + return CompileFunction(isolate, instance, func_index); +} + +const wasm::WasmCode* LazyCompilationOrchestrator::CompileDirectCall( + Isolate* isolate, Handle<WasmInstanceObject> instance, + Maybe<uint32_t> maybe_func_to_return_idx, const wasm::WasmCode* wasm_caller, + int call_offset) { + struct WasmDirectCallData { + uint32_t offset = 0; + uint32_t func_index = 0; + }; + std::vector<Maybe<WasmDirectCallData>> non_compiled_functions; + Decoder decoder(nullptr, nullptr); + { + DisallowHeapAllocation no_gc; + Handle<WasmCompiledModule> caller_module( + wasm_caller->owner()->compiled_module(), isolate); + SeqOneByteString* module_bytes = caller_module->module_bytes(); + uint32_t caller_func_index = wasm_caller->index(); + SourcePositionTableIterator source_pos_iterator( + Handle<ByteArray>(ByteArray::cast( + caller_module->source_positions()->get(caller_func_index)))); + + const byte* func_bytes = + module_bytes->GetChars() + + caller_module->module()->functions[caller_func_index].code.offset(); + for (RelocIterator it(wasm_caller->instructions(), + wasm_caller->reloc_info(), + wasm_caller->constant_pool(), + RelocInfo::ModeMask(RelocInfo::WASM_CALL)); + !it.done(); it.next()) { + const WasmCode* callee = isolate->wasm_code_manager()->LookupCode( + it.rinfo()->target_address()); + if (callee->kind() != WasmCode::LazyStub) { + non_compiled_functions.push_back(Nothing<WasmDirectCallData>()); + continue; + } + // TODO(clemensh): Introduce safe_cast<T, bool> which (D)CHECKS + // (depending on the bool) against limits of T and then static_casts. + size_t offset_l = it.rinfo()->pc() - wasm_caller->instructions().start(); + DCHECK_GE(kMaxInt, offset_l); + int offset = static_cast<int>(offset_l); + int byte_pos = + AdvanceSourcePositionTableIterator(source_pos_iterator, offset); + uint32_t called_func_index = + ExtractDirectCallIndex(decoder, func_bytes + byte_pos); + DCHECK_LT(called_func_index, + caller_module->GetNativeModule()->FunctionCount()); + WasmDirectCallData data; + data.offset = offset; + data.func_index = called_func_index; + non_compiled_functions.push_back(Just<WasmDirectCallData>(data)); + // Call offset one instruction after the call. Remember the last called + // function before that offset. + if (offset < call_offset) { + maybe_func_to_return_idx = Just(called_func_index); + } + } } + uint32_t func_to_return_idx = maybe_func_to_return_idx.ToChecked(); - Code* ret = - Code::cast(compiled_module->code_table()->get(func_to_return_idx)); - DCHECK_EQ(Code::WASM_FUNCTION, ret->kind()); - return handle(ret, isolate); + TRACE_LAZY( + "Starting lazy compilation (func %u @%d, js_to_wasm: false, patch " + "caller: true). \n", + func_to_return_idx, call_offset); + + // TODO(clemensh): compile all functions in non_compiled_functions in + // background, wait for func_to_return_idx. + CompileFunction(isolate, instance, func_to_return_idx); + + Handle<WasmCompiledModule> compiled_module(instance->compiled_module(), + isolate); + WasmCode* ret = + compiled_module->GetNativeModule()->GetCode(func_to_return_idx); + + DCHECK_NOT_NULL(ret); + { + DisallowHeapAllocation no_gc; + // Now patch the code object with all functions which are now compiled. This + // will pick up any other compiled functions, not only {ret}. + size_t idx = 0; + size_t patched = 0; + for (RelocIterator + it(wasm_caller->instructions(), wasm_caller->reloc_info(), + wasm_caller->constant_pool(), + RelocInfo::ModeMask(RelocInfo::WASM_CALL)); + !it.done(); it.next(), ++idx) { + auto& info = non_compiled_functions[idx]; + if (info.IsNothing()) continue; + uint32_t lookup = info.ToChecked().func_index; + const WasmCode* callee_compiled = + compiled_module->GetNativeModule()->GetCode(lookup); + if (callee_compiled->kind() != WasmCode::Function) continue; + it.rinfo()->set_wasm_call_address( + isolate, callee_compiled->instructions().start()); + ++patched; + } + DCHECK_EQ(non_compiled_functions.size(), idx); + TRACE_LAZY("Patched %zu location(s) in the caller.\n", patched); + } + return ret; } ModuleCompiler::CodeGenerationSchedule::CodeGenerationSchedule( @@ -942,7 +1357,8 @@ size_t ModuleCompiler::CodeGenerationSchedule::GetRandomIndexInSchedule() { } ModuleCompiler::ModuleCompiler(Isolate* isolate, WasmModule* module, - Handle<Code> centry_stub) + Handle<Code> centry_stub, + wasm::NativeModule* native_module) : isolate_(isolate), module_(module), async_counters_(isolate->async_counters()), @@ -956,7 +1372,8 @@ ModuleCompiler::ModuleCompiler(Isolate* isolate, WasmModule* module, Min(static_cast<size_t>(FLAG_wasm_num_compilation_tasks), V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads())), stopped_compilation_tasks_(num_background_tasks_), - centry_stub_(centry_stub) {} + centry_stub_(centry_stub), + native_module_(native_module) {} // The actual runnable task that performs compilations in the background. void ModuleCompiler::OnBackgroundTaskStopped() { @@ -965,7 +1382,7 @@ void ModuleCompiler::OnBackgroundTaskStopped() { DCHECK_LE(stopped_compilation_tasks_, num_background_tasks_); } -// Run by each compilation task The no_finisher_callback is called +// Run by each compilation task. The no_finisher_callback is called // within the result_mutex_ lock when no finishing task is running, // i.e. when the finisher_is_running_ flag is not set. bool ModuleCompiler::FetchAndExecuteCompilationUnit( @@ -1009,18 +1426,22 @@ size_t ModuleCompiler::InitializeCompilationUnits( Vector<const uint8_t> bytes(wire_bytes.start() + func->code.offset(), func->code.end_offset() - func->code.offset()); WasmName name = wire_bytes.GetName(func); - builder.AddUnit(module_env, func, buffer_offset, bytes, name); + DCHECK_IMPLIES(FLAG_wasm_jit_to_native, native_module_ != nullptr); + builder.AddUnit(module_env, native_module_, func, buffer_offset, bytes, + name); } builder.Commit(); return funcs_to_compile; } void ModuleCompiler::RestartCompilationTasks() { + v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate_); + std::shared_ptr<v8::TaskRunner> task_runner = + V8::GetCurrentPlatform()->GetBackgroundTaskRunner(v8_isolate); + base::LockGuard<base::Mutex> guard(&tasks_mutex_); for (; stopped_compilation_tasks_ > 0; --stopped_compilation_tasks_) { - V8::GetCurrentPlatform()->CallOnBackgroundThread( - new CompilationTask(this), - v8::Platform::ExpectedRuntime::kShortRunningTask); + task_runner->PostTask(base::make_unique<CompilationTask>(this)); } } @@ -1029,12 +1450,14 @@ size_t ModuleCompiler::FinishCompilationUnits( size_t finished = 0; while (true) { int func_index = -1; - MaybeHandle<Code> result = FinishCompilationUnit(thrower, &func_index); + WasmCodeWrapper result = FinishCompilationUnit(thrower, &func_index); if (func_index < 0) break; ++finished; DCHECK_IMPLIES(result.is_null(), thrower->error()); if (result.is_null()) break; - results[func_index] = result.ToHandleChecked(); + if (result.IsCodeObject()) { + results[func_index] = result.GetCode(); + } } bool do_restart; { @@ -1050,8 +1473,8 @@ void ModuleCompiler::SetFinisherIsRunning(bool value) { finisher_is_running_ = value; } -MaybeHandle<Code> ModuleCompiler::FinishCompilationUnit(ErrorThrower* thrower, - int* func_index) { +WasmCodeWrapper ModuleCompiler::FinishCompilationUnit(ErrorThrower* thrower, + int* func_index) { std::unique_ptr<compiler::WasmCompilationUnit> unit; { base::LockGuard<base::Mutex> guard(&result_mutex_); @@ -1132,15 +1555,17 @@ void ModuleCompiler::CompileSequentially(const ModuleWireBytes& wire_bytes, if (func.imported) continue; // Imports are compiled at instantiation time. // Compile the function. - MaybeHandle<Code> code = compiler::WasmCompilationUnit::CompileWasmFunction( - thrower, isolate_, wire_bytes, module_env, &func); + WasmCodeWrapper code = compiler::WasmCompilationUnit::CompileWasmFunction( + native_module_, thrower, isolate_, wire_bytes, module_env, &func); if (code.is_null()) { TruncatedUserString<> name(wire_bytes.GetName(&func)); thrower->CompileError("Compilation of #%d:%.*s failed.", i, name.length(), name.start()); break; } - results[i] = code.ToHandleChecked(); + if (code.IsCodeObject()) { + results[i] = code.GetCode(); + } } } @@ -1175,7 +1600,13 @@ MaybeHandle<WasmModuleObject> ModuleCompiler::CompileToModuleObject( const ModuleWireBytes& wire_bytes, Handle<Script> asm_js_script, Vector<const byte> asm_js_offset_table_bytes) { Handle<Code> centry_stub = CEntryStub(isolate, 1).GetCode(); - ModuleCompiler compiler(isolate, module.get(), centry_stub); + // TODO(mtrofin): the wasm::NativeModule parameter to the ModuleCompiler + // constructor is null here, and initialized in CompileToModuleObjectInternal. + // This is a point-in-time, until we remove the FLAG_wasm_jit_to_native flag, + // and stop needing a FixedArray for code for the non-native case. Otherwise, + // we end up moving quite a bit of initialization logic here that is also + // needed in CompileToModuleObjectInternal, complicating the change. + ModuleCompiler compiler(isolate, module.get(), centry_stub, nullptr); return compiler.CompileToModuleObjectInternal(thrower, std::move(module), wire_bytes, asm_js_script, asm_js_offset_table_bytes); @@ -1187,9 +1618,19 @@ bool compile_lazy(const WasmModule* module) { (FLAG_asm_wasm_lazy_compilation && module->is_asm_js()); } -void FlushICache(Isolate* isolate, Handle<FixedArray> code_table) { - for (int i = 0; i < code_table->length(); ++i) { - Handle<Code> code = code_table->GetValueChecked<Code>(isolate, i); +void FlushICache(Isolate* isolate, const wasm::NativeModule* native_module) { + for (uint32_t i = 0, e = native_module->FunctionCount(); i < e; ++i) { + const wasm::WasmCode* code = native_module->GetCode(i); + if (code == nullptr) continue; + Assembler::FlushICache(isolate, code->instructions().start(), + code->instructions().size()); + } +} + +void FlushICache(Isolate* isolate, Handle<FixedArray> functions) { + for (int i = 0, e = functions->length(); i < e; ++i) { + if (!functions->get(i)->IsCode()) continue; + Code* code = Code::cast(functions->get(i)); Assembler::FlushICache(isolate, code->instruction_start(), code->instruction_size()); } @@ -1199,11 +1640,26 @@ byte* raw_buffer_ptr(MaybeHandle<JSArrayBuffer> buffer, int offset) { return static_cast<byte*>(buffer.ToHandleChecked()->backing_store()) + offset; } -void RecordStats(Code* code, Counters* counters) { +void RecordStats(const Code* code, Counters* counters) { counters->wasm_generated_code_size()->Increment(code->body_size()); counters->wasm_reloc_size()->Increment(code->relocation_info()->length()); } +void RecordStats(const wasm::WasmCode* code, Counters* counters) { + counters->wasm_generated_code_size()->Increment( + static_cast<int>(code->instructions().size())); + counters->wasm_reloc_size()->Increment( + static_cast<int>(code->reloc_info().size())); +} + +void RecordStats(WasmCodeWrapper wrapper, Counters* counters) { + if (wrapper.IsCodeObject()) { + RecordStats(*wrapper.GetCode(), counters); + } else { + RecordStats(wrapper.GetWasmCode(), counters); + } +} + void RecordStats(Handle<FixedArray> functions, Counters* counters) { DisallowHeapAllocation no_gc; for (int i = 0; i < functions->length(); ++i) { @@ -1211,123 +1667,163 @@ void RecordStats(Handle<FixedArray> functions, Counters* counters) { if (val->IsCode()) RecordStats(Code::cast(val), counters); } } -Handle<Script> CreateWasmScript(Isolate* isolate, - const ModuleWireBytes& wire_bytes) { - Handle<Script> script = - isolate->factory()->NewScript(isolate->factory()->empty_string()); - script->set_context_data(isolate->native_context()->debug_context_id()); - script->set_type(Script::TYPE_WASM); - - int hash = StringHasher::HashSequentialString( - reinterpret_cast<const char*>(wire_bytes.start()), - static_cast<int>(wire_bytes.length()), kZeroHashSeed); - const int kBufferSize = 32; - char buffer[kBufferSize]; - int url_chars = SNPrintF(ArrayVector(buffer), "wasm://wasm/%08x", hash); - DCHECK(url_chars >= 0 && url_chars < kBufferSize); - MaybeHandle<String> url_str = isolate->factory()->NewStringFromOneByte( - Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), url_chars), - TENURED); - script->set_source_url(*url_str.ToHandleChecked()); - - int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash); - DCHECK(name_chars >= 0 && name_chars < kBufferSize); - MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte( - Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), name_chars), - TENURED); - script->set_name(*name_str.ToHandleChecked()); - - return script; +void RecordStats(const wasm::NativeModule* native_module, Counters* counters) { + for (uint32_t i = 0, e = native_module->FunctionCount(); i < e; ++i) { + const wasm::WasmCode* code = native_module->GetCode(i); + if (code != nullptr) RecordStats(code, counters); + } } // Ensure that the code object in <code_table> at offset <func_index> has // deoptimization data attached. This is needed for lazy compile stubs which are // called from JS_TO_WASM functions or via exported function tables. The deopt // data is used to determine which function this lazy compile stub belongs to. -Handle<Code> EnsureExportedLazyDeoptData(Isolate* isolate, - Handle<WasmInstanceObject> instance, - Handle<FixedArray> code_table, - int func_index) { - Handle<Code> code(Code::cast(code_table->get(func_index)), isolate); - if (code->builtin_index() != Builtins::kWasmCompileLazy) { - // No special deopt data needed for compiled functions, and imported - // functions, which map to Illegal at this point (they get compiled at - // instantiation time). - DCHECK(code->kind() == Code::WASM_FUNCTION || - code->kind() == Code::WASM_TO_JS_FUNCTION || - code->builtin_index() == Builtins::kIllegal); - return code; +// TODO(mtrofin): remove the instance and code_table members once we remove the +// FLAG_wasm_jit_to_native +WasmCodeWrapper EnsureExportedLazyDeoptData(Isolate* isolate, + Handle<WasmInstanceObject> instance, + Handle<FixedArray> code_table, + wasm::NativeModule* native_module, + uint32_t func_index) { + if (!FLAG_wasm_jit_to_native) { + Handle<Code> code(Code::cast(code_table->get(func_index)), isolate); + if (code->builtin_index() != Builtins::kWasmCompileLazy) { + // No special deopt data needed for compiled functions, and imported + // functions, which map to Illegal at this point (they get compiled at + // instantiation time). + DCHECK(code->kind() == Code::WASM_FUNCTION || + code->kind() == Code::WASM_TO_JS_FUNCTION || + code->kind() == Code::WASM_TO_WASM_FUNCTION || + code->builtin_index() == Builtins::kIllegal); + return WasmCodeWrapper(code); + } + + // deopt_data: + // #0: weak instance + // #1: func_index + // might be extended later for table exports (see + // EnsureTableExportLazyDeoptData). + Handle<FixedArray> deopt_data(code->deoptimization_data()); + DCHECK_EQ(0, deopt_data->length() % 2); + if (deopt_data->length() == 0) { + code = isolate->factory()->CopyCode(code); + code_table->set(func_index, *code); + AttachWasmFunctionInfo(isolate, code, instance, func_index); + } +#ifdef DEBUG + auto func_info = GetWasmFunctionInfo(isolate, code); + DCHECK_IMPLIES(!instance.is_null(), + *func_info.instance.ToHandleChecked() == *instance); + DCHECK_EQ(func_index, func_info.func_index); +#endif + return WasmCodeWrapper(code); + } else { + wasm::WasmCode* code = native_module->GetCode(func_index); + // {code} will be nullptr when exporting imports. + if (code == nullptr || code->kind() != wasm::WasmCode::LazyStub || + !code->IsAnonymous()) { + return WasmCodeWrapper(code); + } + // Clone the lazy builtin into the native module. + return WasmCodeWrapper(native_module->CloneLazyBuiltinInto(func_index)); } - // deopt_data: - // #0: weak instance - // #1: func_index - // might be extended later for table exports (see - // EnsureTableExportLazyDeoptData). - Handle<FixedArray> deopt_data(code->deoptimization_data()); - DCHECK_EQ(0, deopt_data->length() % 2); - if (deopt_data->length() == 0) { - code = isolate->factory()->CopyCode(code); - code_table->set(func_index, *code); - deopt_data = isolate->factory()->NewFixedArray(2, TENURED); - code->set_deoptimization_data(*deopt_data); - if (!instance.is_null()) { - Handle<WeakCell> weak_instance = - isolate->factory()->NewWeakCell(instance); - deopt_data->set(0, *weak_instance); - } - deopt_data->set(1, Smi::FromInt(func_index)); - } - DCHECK_IMPLIES(!instance.is_null(), - WeakCell::cast(code->deoptimization_data()->get(0))->value() == - *instance); - DCHECK_EQ(func_index, Smi::ToInt(code->deoptimization_data()->get(1))); - return code; } // Ensure that the code object in <code_table> at offset <func_index> has // deoptimization data attached. This is needed for lazy compile stubs which are // called from JS_TO_WASM functions or via exported function tables. The deopt // data is used to determine which function this lazy compile stub belongs to. -Handle<Code> EnsureTableExportLazyDeoptData( +// TODO(mtrofin): remove the instance and code_table members once we remove the +// FLAG_wasm_jit_to_native +WasmCodeWrapper EnsureTableExportLazyDeoptData( Isolate* isolate, Handle<WasmInstanceObject> instance, - Handle<FixedArray> code_table, int func_index, - Handle<FixedArray> export_table, int export_index, - std::unordered_map<uint32_t, uint32_t>& table_export_count) { - Handle<Code> code = - EnsureExportedLazyDeoptData(isolate, instance, code_table, func_index); - if (code->builtin_index() != Builtins::kWasmCompileLazy) return code; - - // deopt_data: - // #0: weak instance - // #1: func_index - // [#2: export table - // #3: export table index] - // [#4: export table - // #5: export table index] - // ... - // table_export_count counts down and determines the index for the new export - // table entry. - auto table_export_entry = table_export_count.find(func_index); - DCHECK(table_export_entry != table_export_count.end()); - DCHECK_LT(0, table_export_entry->second); - uint32_t this_idx = 2 * table_export_entry->second; - --table_export_entry->second; - Handle<FixedArray> deopt_data(code->deoptimization_data()); - DCHECK_EQ(0, deopt_data->length() % 2); - if (deopt_data->length() == 2) { - // Then only the "header" (#0 and #1) exists. Extend for the export table - // entries (make space for this_idx + 2 elements). - deopt_data = isolate->factory()->CopyFixedArrayAndGrow(deopt_data, this_idx, - TENURED); - code->set_deoptimization_data(*deopt_data); - } - DCHECK_LE(this_idx + 2, deopt_data->length()); - DCHECK(deopt_data->get(this_idx)->IsUndefined(isolate)); - DCHECK(deopt_data->get(this_idx + 1)->IsUndefined(isolate)); - deopt_data->set(this_idx, *export_table); - deopt_data->set(this_idx + 1, Smi::FromInt(export_index)); - return code; + Handle<FixedArray> code_table, wasm::NativeModule* native_module, + uint32_t func_index, Handle<FixedArray> export_table, int export_index, + std::unordered_map<uint32_t, uint32_t>* table_export_count) { + if (!FLAG_wasm_jit_to_native) { + Handle<Code> code = + EnsureExportedLazyDeoptData(isolate, instance, code_table, + native_module, func_index) + .GetCode(); + if (code->builtin_index() != Builtins::kWasmCompileLazy) + return WasmCodeWrapper(code); + + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(isolate->heap()); + + // deopt_data: + // #0: weak instance + // #1: func_index + // [#2: export table + // #3: export table index] + // [#4: export table + // #5: export table index] + // ... + // table_export_count counts down and determines the index for the new + // export table entry. + auto table_export_entry = table_export_count->find(func_index); + DCHECK(table_export_entry != table_export_count->end()); + DCHECK_LT(0, table_export_entry->second); + uint32_t this_idx = 2 * table_export_entry->second; + --table_export_entry->second; + Handle<FixedArray> deopt_data(code->deoptimization_data()); + DCHECK_EQ(0, deopt_data->length() % 2); + if (deopt_data->length() == 2) { + // Then only the "header" (#0 and #1) exists. Extend for the export table + // entries (make space for this_idx + 2 elements). + deopt_data = isolate->factory()->CopyFixedArrayAndGrow(deopt_data, + this_idx, TENURED); + code->set_deoptimization_data(*deopt_data); + } + DCHECK_LE(this_idx + 2, deopt_data->length()); + DCHECK(deopt_data->get(this_idx)->IsUndefined(isolate)); + DCHECK(deopt_data->get(this_idx + 1)->IsUndefined(isolate)); + deopt_data->set(this_idx, *export_table); + deopt_data->set(this_idx + 1, Smi::FromInt(export_index)); + return WasmCodeWrapper(code); + } else { + const wasm::WasmCode* code = + EnsureExportedLazyDeoptData(isolate, instance, code_table, + native_module, func_index) + .GetWasmCode(); + if (code == nullptr || code->kind() != wasm::WasmCode::LazyStub) + return WasmCodeWrapper(code); + + // deopt_data: + // [#0: export table + // #1: export table index] + // [#2: export table + // #3: export table index] + // ... + // table_export_count counts down and determines the index for the new + // export table entry. + auto table_export_entry = table_export_count->find(func_index); + DCHECK(table_export_entry != table_export_count->end()); + DCHECK_LT(0, table_export_entry->second); + --table_export_entry->second; + uint32_t this_idx = 2 * table_export_entry->second; + int int_func_index = static_cast<int>(func_index); + Object* deopt_entry = + native_module->compiled_module()->lazy_compile_data()->get( + int_func_index); + FixedArray* deopt_data = nullptr; + if (!deopt_entry->IsFixedArray()) { + // we count indices down, so we enter here first for the + // largest index. + deopt_data = *isolate->factory()->NewFixedArray(this_idx + 2, TENURED); + native_module->compiled_module()->lazy_compile_data()->set(int_func_index, + deopt_data); + } else { + deopt_data = FixedArray::cast(deopt_entry); + DCHECK_LE(this_idx + 2, deopt_data->length()); + } + DCHECK(deopt_data->get(this_idx)->IsUndefined(isolate)); + DCHECK(deopt_data->get(this_idx + 1)->IsUndefined(isolate)); + deopt_data->set(this_idx, *export_table); + deopt_data->set(this_idx + 1, Smi::FromInt(export_index)); + return WasmCodeWrapper(code); + } } bool in_bounds(uint32_t offset, uint32_t size, uint32_t upper) { @@ -1337,47 +1833,66 @@ bool in_bounds(uint32_t offset, uint32_t size, uint32_t upper) { using WasmInstanceMap = IdentityMap<Handle<WasmInstanceObject>, FreeStoreAllocationPolicy>; -Handle<Code> UnwrapExportOrCompileImportWrapper( - Isolate* isolate, int index, FunctionSig* sig, Handle<JSReceiver> target, - ModuleOrigin origin, WasmInstanceMap* imported_instances, - Handle<FixedArray> js_imports_table, Handle<WasmInstanceObject> instance) { - WasmFunction* other_func = GetWasmFunctionForExport(isolate, target); - if (other_func) { - if (!sig->Equals(other_func->sig)) return Handle<Code>::null(); - // Signature matched. Unwrap the import wrapper and return the raw wasm - // function code. - // Remember the wasm instance of the import. We have to keep it alive. - Handle<WasmInstanceObject> imported_instance( - Handle<WasmExportedFunction>::cast(target)->instance(), isolate); - imported_instances->Set(imported_instance, imported_instance); - Handle<Code> wasm_code = - UnwrapExportWrapper(Handle<JSFunction>::cast(target)); - // Create a WasmToWasm wrapper to replace the current wasm context with - // the imported_instance one, in order to access the right memory. - // If the imported instance does not have memory, avoid the wrapper. - // TODO(wasm): Avoid the wrapper also if instance memory and imported - // instance share the same memory object. - bool needs_wasm_to_wasm_wrapper = imported_instance->has_memory_object(); - if (!needs_wasm_to_wasm_wrapper) return wasm_code; - Address new_wasm_context = - reinterpret_cast<Address>(imported_instance->wasm_context()); +WasmCodeWrapper MakeWasmToWasmWrapper( + Isolate* isolate, Handle<WasmExportedFunction> imported_function, + FunctionSig* expected_sig, FunctionSig** sig, + WasmInstanceMap* imported_instances, Handle<WasmInstanceObject> instance, + uint32_t index) { + // TODO(wasm): cache WASM-to-WASM wrappers by signature and clone+patch. + Handle<WasmInstanceObject> imported_instance(imported_function->instance(), + isolate); + imported_instances->Set(imported_instance, imported_instance); + WasmContext* new_wasm_context = imported_instance->wasm_context()->get(); + Address new_wasm_context_address = + reinterpret_cast<Address>(new_wasm_context); + *sig = imported_instance->module() + ->functions[imported_function->function_index()] + .sig; + if (expected_sig && !expected_sig->Equals(*sig)) return {}; + + if (!FLAG_wasm_jit_to_native) { Handle<Code> wrapper_code = compiler::CompileWasmToWasmWrapper( - isolate, wasm_code, sig, index, new_wasm_context); - // Set the deoptimization data for the WasmToWasm wrapper. - // TODO(wasm): Remove the deoptimization data when we will use tail calls - // for WasmToWasm wrappers. - Factory* factory = isolate->factory(); - Handle<WeakCell> weak_link = factory->NewWeakCell(instance); - Handle<FixedArray> deopt_data = factory->NewFixedArray(2, TENURED); - deopt_data->set(0, *weak_link); - deopt_data->set(1, Smi::FromInt(index)); - wrapper_code->set_deoptimization_data(*deopt_data); - return wrapper_code; + isolate, imported_function->GetWasmCode(), *sig, + new_wasm_context_address); + // Set the deoptimization data for the WasmToWasm wrapper. This is + // needed by the interpreter to find the imported instance for + // a cross-instance call. + AttachWasmFunctionInfo(isolate, wrapper_code, imported_instance, + imported_function->function_index()); + return WasmCodeWrapper(wrapper_code); + } else { + Handle<Code> code = compiler::CompileWasmToWasmWrapper( + isolate, imported_function->GetWasmCode(), *sig, + new_wasm_context_address); + return WasmCodeWrapper( + instance->compiled_module()->GetNativeModule()->AddCodeCopy( + code, wasm::WasmCode::WasmToWasmWrapper, index)); + } +} + +WasmCodeWrapper UnwrapExportOrCompileImportWrapper( + Isolate* isolate, FunctionSig* sig, Handle<JSReceiver> target, + uint32_t import_index, ModuleOrigin origin, + WasmInstanceMap* imported_instances, Handle<FixedArray> js_imports_table, + Handle<WasmInstanceObject> instance) { + if (WasmExportedFunction::IsWasmExportedFunction(*target)) { + FunctionSig* unused = nullptr; + return MakeWasmToWasmWrapper( + isolate, Handle<WasmExportedFunction>::cast(target), sig, &unused, + imported_instances, instance, import_index); } // No wasm function or being debugged. Compile a new wrapper for the new // signature. - return compiler::CompileWasmToJSWrapper(isolate, target, sig, index, origin, - js_imports_table); + if (FLAG_wasm_jit_to_native) { + Handle<Code> temp_code = compiler::CompileWasmToJSWrapper( + isolate, target, sig, import_index, origin, js_imports_table); + return WasmCodeWrapper( + instance->compiled_module()->GetNativeModule()->AddCodeCopy( + temp_code, wasm::WasmCode::WasmToJsWrapper, import_index)); + } else { + return WasmCodeWrapper(compiler::CompileWasmToJSWrapper( + isolate, target, sig, import_index, origin, js_imports_table)); + } } double MonotonicallyIncreasingTimeInMs() { @@ -1394,7 +1909,6 @@ std::unique_ptr<compiler::ModuleEnv> CreateDefaultModuleEnv( Isolate* isolate, WasmModule* module, Handle<Code> illegal_builtin) { std::vector<GlobalHandleAddress> function_tables; std::vector<GlobalHandleAddress> signature_tables; - std::vector<SignatureMap*> signature_maps; for (size_t i = 0; i < module->function_tables.size(); i++) { Handle<Object> func_table = @@ -1409,7 +1923,6 @@ std::unique_ptr<compiler::ModuleEnv> CreateDefaultModuleEnv( v8::WeakCallbackType::kFinalizer); function_tables.push_back(func_table.address()); signature_tables.push_back(sig_table.address()); - signature_maps.push_back(&module->function_tables[i].map); } std::vector<Handle<Code>> empty_code; @@ -1418,20 +1931,20 @@ std::unique_ptr<compiler::ModuleEnv> CreateDefaultModuleEnv( module, // -- function_tables, // -- signature_tables, // -- - signature_maps, // -- empty_code, // -- - illegal_builtin, // -- - 0 // -- + illegal_builtin // -- }; return std::unique_ptr<compiler::ModuleEnv>(new compiler::ModuleEnv(result)); } -Handle<WasmCompiledModule> NewCompiledModule( - Isolate* isolate, Handle<WasmSharedModuleData> shared, - Handle<FixedArray> code_table, Handle<FixedArray> export_wrappers, - compiler::ModuleEnv* env) { +// TODO(mtrofin): remove code_table when we don't need FLAG_wasm_jit_to_native +Handle<WasmCompiledModule> NewCompiledModule(Isolate* isolate, + WasmModule* module, + Handle<FixedArray> code_table, + Handle<FixedArray> export_wrappers, + compiler::ModuleEnv* env) { Handle<WasmCompiledModule> compiled_module = - WasmCompiledModule::New(isolate, shared, code_table, export_wrappers, + WasmCompiledModule::New(isolate, module, code_table, export_wrappers, env->function_tables, env->signature_tables); return compiled_module; } @@ -1453,7 +1966,11 @@ MaybeHandle<WasmModuleObject> ModuleCompiler::CompileToModuleObjectInternal( TimedHistogramScope wasm_compile_module_time_scope( module_->is_wasm() ? counters()->wasm_compile_wasm_module_time() : counters()->wasm_compile_asm_module_time()); - // The {module> parameter is passed in to transfer ownership of the WasmModule + // TODO(6792): No longer needed once WebAssembly code is off heap. Use + // base::Optional to be able to close the scope before notifying the debugger. + base::Optional<CodeSpaceMemoryModificationScope> modification_scope( + base::in_place_t(), isolate_->heap()); + // The {module} parameter is passed in to transfer ownership of the WasmModule // to this function. The WasmModule itself existed already as an instance // variable of the ModuleCompiler. We check here that the parameter and the // instance variable actually point to the same object. @@ -1462,69 +1979,6 @@ MaybeHandle<WasmModuleObject> ModuleCompiler::CompileToModuleObjectInternal( bool lazy_compile = compile_lazy(module_); Factory* factory = isolate_->factory(); - - // If lazy compile: Initialize the code table with the lazy compile builtin. - // Otherwise: Initialize with the illegal builtin. All call sites will be - // patched at instantiation. - Handle<Code> init_builtin = lazy_compile - ? BUILTIN_CODE(isolate_, WasmCompileLazy) - : BUILTIN_CODE(isolate_, Illegal); - - auto env = CreateDefaultModuleEnv(isolate_, module_, init_builtin); - - // The {code_table} array contains import wrappers and functions (which - // are both included in {functions.size()}, and export wrappers). - int code_table_size = static_cast<int>(module_->functions.size()); - int export_wrappers_size = static_cast<int>(module_->num_exported_functions); - Handle<FixedArray> code_table = - factory->NewFixedArray(static_cast<int>(code_table_size), TENURED); - Handle<FixedArray> export_wrappers = - factory->NewFixedArray(static_cast<int>(export_wrappers_size), TENURED); - // Initialize the code table. - for (int i = 0, e = code_table->length(); i < e; ++i) { - code_table->set(i, *init_builtin); - } - - for (int i = 0, e = export_wrappers->length(); i < e; ++i) { - export_wrappers->set(i, *init_builtin); - } - - if (!lazy_compile) { - size_t funcs_to_compile = - module_->functions.size() - module_->num_imported_functions; - bool compile_parallel = - !FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks > 0 && - funcs_to_compile > 1 && - V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads() > 0; - // Avoid a race condition by collecting results into a second vector. - std::vector<Handle<Code>> results(env->module->functions.size()); - - if (compile_parallel) { - CompileInParallel(wire_bytes, env.get(), results, thrower); - } else { - CompileSequentially(wire_bytes, env.get(), results, thrower); - } - if (thrower->error()) return {}; - - // At this point, compilation has completed. Update the code table. - for (size_t i = - module_->num_imported_functions + FLAG_skip_compiling_wasm_funcs; - i < results.size(); ++i) { - Code* code = *results[i]; - code_table->set(static_cast<int>(i), code); - RecordStats(code, counters()); - } - } else if (module_->is_wasm()) { - // Validate wasm modules for lazy compilation. Don't validate asm.js - // modules, they are valid by construction (otherwise a CHECK will fail - // during lazy compilation). - // TODO(clemensh): According to the spec, we can actually skip validation - // at module creation time, and return a function that always traps at - // (lazy) compilation time. - ValidateSequentially(wire_bytes, env.get(), thrower); - } - if (thrower->error()) return {}; - // Create heap objects for script, module bytes and asm.js offset table to // be stored in the shared module data. Handle<Script> script; @@ -1562,34 +2016,104 @@ MaybeHandle<WasmModuleObject> ModuleCompiler::CompileToModuleObjectInternal( script, asm_js_offset_table); if (lazy_compile) WasmSharedModuleData::PrepareForLazyCompilation(shared); + Handle<Code> init_builtin = lazy_compile + ? BUILTIN_CODE(isolate_, WasmCompileLazy) + : BUILTIN_CODE(isolate_, Illegal); + + // TODO(mtrofin): remove code_table and code_table_size when we don't + // need FLAG_wasm_jit_to_native anymore. Keep export_wrappers. + int code_table_size = static_cast<int>(module_->functions.size()); + int export_wrappers_size = static_cast<int>(module_->num_exported_functions); + Handle<FixedArray> code_table = + factory->NewFixedArray(static_cast<int>(code_table_size), TENURED); + Handle<FixedArray> export_wrappers = + factory->NewFixedArray(static_cast<int>(export_wrappers_size), TENURED); + // Initialize the code table. + for (int i = 0, e = code_table->length(); i < e; ++i) { + code_table->set(i, *init_builtin); + } + + for (int i = 0, e = export_wrappers->length(); i < e; ++i) { + export_wrappers->set(i, *init_builtin); + } + auto env = CreateDefaultModuleEnv(isolate_, module_, init_builtin); + // Create the compiled module object and populate with compiled functions // and information needed at instantiation time. This object needs to be // serializable. Instantiation may occur off a deserialized version of this // object. Handle<WasmCompiledModule> compiled_module = NewCompiledModule( - isolate_, shared, code_table, export_wrappers, env.get()); + isolate_, shared->module(), code_table, export_wrappers, env.get()); + native_module_ = compiled_module->GetNativeModule(); + compiled_module->OnWasmModuleDecodingComplete(shared); + if (lazy_compile && FLAG_wasm_jit_to_native) { + compiled_module->set_lazy_compile_data(isolate_->factory()->NewFixedArray( + static_cast<int>(module_->functions.size()), TENURED)); + } + + if (!lazy_compile) { + size_t funcs_to_compile = + module_->functions.size() - module_->num_imported_functions; + bool compile_parallel = + !FLAG_trace_wasm_decoder && FLAG_wasm_num_compilation_tasks > 0 && + funcs_to_compile > 1 && + V8::GetCurrentPlatform()->NumberOfAvailableBackgroundThreads() > 0; + // Avoid a race condition by collecting results into a second vector. + std::vector<Handle<Code>> results( + FLAG_wasm_jit_to_native ? 0 : env->module->functions.size()); + + if (compile_parallel) { + CompileInParallel(wire_bytes, env.get(), results, thrower); + } else { + CompileSequentially(wire_bytes, env.get(), results, thrower); + } + if (thrower->error()) return {}; + + if (!FLAG_wasm_jit_to_native) { + // At this point, compilation has completed. Update the code table. + for (size_t i = + module_->num_imported_functions + FLAG_skip_compiling_wasm_funcs; + i < results.size(); ++i) { + Code* code = *results[i]; + code_table->set(static_cast<int>(i), code); + RecordStats(code, counters()); + } + } else { + RecordStats(native_module_, counters()); + } + } else { + if (module_->is_wasm()) { + // Validate wasm modules for lazy compilation. Don't validate asm.js + // modules, they are valid by construction (otherwise a CHECK will fail + // during lazy compilation). + // TODO(clemensh): According to the spec, we can actually skip validation + // at module creation time, and return a function that always traps at + // (lazy) compilation time. + ValidateSequentially(wire_bytes, env.get(), thrower); + } + if (FLAG_wasm_jit_to_native) { + native_module_->SetLazyBuiltin(BUILTIN_CODE(isolate_, WasmCompileLazy)); + } + } + if (thrower->error()) return {}; + + // Compile JS->wasm wrappers for exported functions. + CompileJsToWasmWrappers(isolate_, compiled_module, counters()); + + Handle<WasmModuleObject> result = + WasmModuleObject::New(isolate_, compiled_module); // If we created a wasm script, finish it now and make it public to the // debugger. if (asm_js_script.is_null()) { + // Close the CodeSpaceMemoryModificationScope before calling into the + // debugger. + modification_scope.reset(); script->set_wasm_compiled_module(*compiled_module); isolate_->debug()->OnAfterCompile(script); } - // Compile JS->wasm wrappers for exported functions. - JSToWasmWrapperCache js_to_wasm_cache; - int wrapper_index = 0; - for (auto exp : module_->export_table) { - if (exp.kind != kExternalFunction) continue; - Handle<Code> wasm_code = EnsureExportedLazyDeoptData( - isolate_, Handle<WasmInstanceObject>::null(), code_table, exp.index); - Handle<Code> wrapper_code = js_to_wasm_cache.CloneOrCompileJSToWasmWrapper( - isolate_, module_, wasm_code, exp.index); - export_wrappers->set(wrapper_index, *wrapper_code); - RecordStats(*wrapper_code, counters()); - ++wrapper_index; - } - return WasmModuleObject::New(isolate_, compiled_module); + return result; } InstanceBuilder::InstanceBuilder( @@ -1621,6 +2145,11 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { SanitizeImports(); if (thrower_->error()) return {}; + // TODO(6792): No longer needed once WebAssembly code is off heap. + // Use base::Optional to be able to close the scope before executing the start + // function. + base::Optional<CodeSpaceMemoryModificationScope> modification_scope( + base::in_place_t(), isolate_->heap()); // From here on, we expect the build pipeline to run without exiting to JS. // Exception is when we run the startup function. DisallowJavascriptExecution no_js(isolate_); @@ -1633,6 +2162,8 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { //-------------------------------------------------------------------------- // Reuse the compiled module (if no owner), otherwise clone. //-------------------------------------------------------------------------- + // TODO(mtrofin): remove code_table and old_code_table + // when FLAG_wasm_jit_to_native is not needed Handle<FixedArray> code_table; Handle<FixedArray> wrapper_table; // We keep around a copy of the old code table, because we'll be replacing @@ -1640,6 +2171,11 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { // able to relocate. Handle<FixedArray> old_code_table; MaybeHandle<WasmInstanceObject> owner; + // native_module is the one we're building now, old_module + // is the one we clone from. They point to the same place if + // we don't need to clone. + wasm::NativeModule* native_module = nullptr; + wasm::NativeModule* old_module = nullptr; TRACE("Starting new module instantiation\n"); { @@ -1665,41 +2201,49 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { // the owner + original state used for cloning and patching // won't be mutated by possible finalizer runs. DCHECK(!owner.is_null()); - TRACE("Cloning from %d\n", original->instance_id()); - old_code_table = original->code_table(); - compiled_module_ = WasmCompiledModule::Clone(isolate_, original); - code_table = compiled_module_->code_table(); - wrapper_table = compiled_module_->export_wrappers(); - // Avoid creating too many handles in the outer scope. - HandleScope scope(isolate_); - - // Clone the code for wasm functions and exports. - for (int i = 0; i < code_table->length(); ++i) { - Handle<Code> orig_code(Code::cast(code_table->get(i)), isolate_); - switch (orig_code->kind()) { - case Code::WASM_TO_JS_FUNCTION: - // Imports will be overwritten with newly compiled wrappers. - break; - case Code::BUILTIN: - DCHECK_EQ(Builtins::kWasmCompileLazy, orig_code->builtin_index()); - // If this code object has deoptimization data, then we need a - // unique copy to attach updated deoptimization data. - if (orig_code->deoptimization_data()->length() > 0) { + if (FLAG_wasm_jit_to_native) { + TRACE("Cloning from %zu\n", original->GetNativeModule()->instance_id); + compiled_module_ = WasmCompiledModule::Clone(isolate_, original); + native_module = compiled_module_->GetNativeModule(); + wrapper_table = compiled_module_->export_wrappers(); + } else { + TRACE("Cloning from %d\n", original->instance_id()); + old_code_table = original->code_table(); + compiled_module_ = WasmCompiledModule::Clone(isolate_, original); + code_table = compiled_module_->code_table(); + wrapper_table = compiled_module_->export_wrappers(); + // Avoid creating too many handles in the outer scope. + HandleScope scope(isolate_); + + // Clone the code for wasm functions and exports. + for (int i = 0; i < code_table->length(); ++i) { + Handle<Code> orig_code(Code::cast(code_table->get(i)), isolate_); + switch (orig_code->kind()) { + case Code::WASM_TO_JS_FUNCTION: + case Code::WASM_TO_WASM_FUNCTION: + // Imports will be overwritten with newly compiled wrappers. + break; + case Code::BUILTIN: + DCHECK_EQ(Builtins::kWasmCompileLazy, orig_code->builtin_index()); + // If this code object has deoptimization data, then we need a + // unique copy to attach updated deoptimization data. + if (orig_code->deoptimization_data()->length() > 0) { + Handle<Code> code = factory->CopyCode(orig_code); + AttachWasmFunctionInfo(isolate_, code, + Handle<WasmInstanceObject>(), i); + code_table->set(i, *code); + } + break; + case Code::WASM_FUNCTION: { Handle<Code> code = factory->CopyCode(orig_code); - Handle<FixedArray> deopt_data = - factory->NewFixedArray(2, TENURED); - deopt_data->set(1, Smi::FromInt(i)); - code->set_deoptimization_data(*deopt_data); + AttachWasmFunctionInfo(isolate_, code, + Handle<WasmInstanceObject>(), i); code_table->set(i, *code); + break; } - break; - case Code::WASM_FUNCTION: { - Handle<Code> code = factory->CopyCode(orig_code); - code_table->set(i, *code); - break; + default: + UNREACHABLE(); } - default: - UNREACHABLE(); } } for (int i = 0; i < wrapper_table->length(); ++i) { @@ -1708,22 +2252,34 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { Handle<Code> code = factory->CopyCode(orig_code); wrapper_table->set(i, *code); } - - RecordStats(code_table, counters()); + if (FLAG_wasm_jit_to_native) { + RecordStats(native_module, counters()); + } else { + RecordStats(code_table, counters()); + } RecordStats(wrapper_table, counters()); } else { // There was no owner, so we can reuse the original. compiled_module_ = original; - old_code_table = factory->CopyFixedArray(compiled_module_->code_table()); - code_table = compiled_module_->code_table(); wrapper_table = compiled_module_->export_wrappers(); - TRACE("Reusing existing instance %d\n", compiled_module_->instance_id()); + if (FLAG_wasm_jit_to_native) { + old_module = compiled_module_->GetNativeModule(); + native_module = old_module; + TRACE("Reusing existing instance %zu\n", + compiled_module_->GetNativeModule()->instance_id); + } else { + old_code_table = + factory->CopyFixedArray(compiled_module_->code_table()); + code_table = compiled_module_->code_table(); + TRACE("Reusing existing instance %d\n", + compiled_module_->instance_id()); + } } compiled_module_->set_native_context(isolate_->native_context()); } //-------------------------------------------------------------------------- - // Allocate the instance object. + // Create the WebAssembly.Instance object. //-------------------------------------------------------------------------- Zone instantiation_zone(isolate_->allocator(), ZONE_NAME); CodeSpecialization code_specialization(isolate_, &instantiation_zone); @@ -1733,6 +2289,7 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { //-------------------------------------------------------------------------- // Set up the globals for the new instance. //-------------------------------------------------------------------------- + WasmContext* wasm_context = instance->wasm_context()->get(); MaybeHandle<JSArrayBuffer> old_globals; uint32_t globals_size = module_->globals_size; if (globals_size > 0) { @@ -1744,22 +2301,13 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { thrower_->RangeError("Out of memory: wasm globals"); return {}; } - Address old_globals_start = compiled_module_->GetGlobalsStartOrNull(); - Address new_globals_start = - static_cast<Address>(global_buffer->backing_store()); - code_specialization.RelocateGlobals(old_globals_start, new_globals_start); - // The address of the backing buffer for the golbals is in native memory - // and, thus, not moving. We need it saved for - // serialization/deserialization purposes - so that the other end - // understands how to relocate the references. We still need to save the - // JSArrayBuffer on the instance, to keep it all alive. - WasmCompiledModule::SetGlobalsStartAddressFrom(factory, compiled_module_, - global_buffer); + wasm_context->globals_start = + reinterpret_cast<byte*>(global_buffer->backing_store()); instance->set_globals_buffer(*global_buffer); } //-------------------------------------------------------------------------- - // Prepare for initialization of function tables. + // Reserve the metadata for indirect function tables. //-------------------------------------------------------------------------- int function_table_count = static_cast<int>(module_->function_tables.size()); table_instances_.reserve(module_->function_tables.size()); @@ -1781,13 +2329,14 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { InitGlobals(); //-------------------------------------------------------------------------- - // Set up the indirect function tables for the new instance. + // Initialize the indirect tables. //-------------------------------------------------------------------------- - if (function_table_count > 0) + if (function_table_count > 0) { InitializeTables(instance, &code_specialization); + } //-------------------------------------------------------------------------- - // Set up the memory for the new instance. + // Allocate the memory array buffer. //-------------------------------------------------------------------------- uint32_t initial_pages = module_->initial_pages; (module_->is_wasm() ? counters()->wasm_wasm_min_mem_pages_count() @@ -1795,18 +2344,45 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { ->AddSample(initial_pages); if (!memory_.is_null()) { - Handle<JSArrayBuffer> memory = memory_.ToHandleChecked(); // Set externally passed ArrayBuffer non neuterable. + Handle<JSArrayBuffer> memory = memory_.ToHandleChecked(); memory->set_is_neuterable(false); DCHECK_IMPLIES(trap_handler::UseTrapHandler(), module_->is_asm_js() || memory->has_guard_region()); } else if (initial_pages > 0) { + // Allocate memory if the initial size is more than 0 pages. memory_ = AllocateMemory(initial_pages); if (memory_.is_null()) return {}; // failed to allocate memory } //-------------------------------------------------------------------------- + // Create the WebAssembly.Memory object. + //-------------------------------------------------------------------------- + if (module_->has_memory) { + if (!instance->has_memory_object()) { + // No memory object exists. Create one. + Handle<WasmMemoryObject> memory_object = WasmMemoryObject::New( + isolate_, memory_, + module_->maximum_pages != 0 ? module_->maximum_pages : -1); + instance->set_memory_object(*memory_object); + } + + // Add the instance object to the list of instances for this memory. + Handle<WasmMemoryObject> memory_object(instance->memory_object(), isolate_); + WasmMemoryObject::AddInstance(isolate_, memory_object, instance); + + if (!memory_.is_null()) { + // Double-check the {memory} array buffer matches the context. + Handle<JSArrayBuffer> memory = memory_.ToHandleChecked(); + uint32_t mem_size = 0; + CHECK(memory->byte_length()->ToUint32(&mem_size)); + CHECK_EQ(wasm_context->mem_size, mem_size); + CHECK_EQ(wasm_context->mem_start, memory->backing_store()); + } + } + + //-------------------------------------------------------------------------- // Check that indirect function table segments are within bounds. //-------------------------------------------------------------------------- for (WasmTableInit& table_init : module_->table_inits) { @@ -1826,78 +2402,44 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { //-------------------------------------------------------------------------- for (WasmDataSegment& seg : module_->data_segments) { uint32_t base = EvalUint32InitExpr(seg.dest_addr); - uint32_t mem_size = 0; - if (!memory_.is_null()) { - CHECK(memory_.ToHandleChecked()->byte_length()->ToUint32(&mem_size)); - } - if (!in_bounds(base, seg.source.length(), mem_size)) { + if (!in_bounds(base, seg.source.length(), wasm_context->mem_size)) { thrower_->LinkError("data segment is out of bounds"); return {}; } } - //-------------------------------------------------------------------------- - // Initialize memory. - //-------------------------------------------------------------------------- - Address mem_start = nullptr; - uint32_t mem_size = 0; - if (!memory_.is_null()) { - Handle<JSArrayBuffer> memory = memory_.ToHandleChecked(); - mem_start = static_cast<Address>(memory->backing_store()); - CHECK(memory->byte_length()->ToUint32(&mem_size)); - LoadDataSegments(mem_start, mem_size); - // Just like with globals, we need to keep both the JSArrayBuffer - // and save the start pointer. - instance->set_memory_buffer(*memory); - } - - //-------------------------------------------------------------------------- - // Create a memory object to have a WasmContext. - //-------------------------------------------------------------------------- - if (module_->has_memory) { - if (!instance->has_memory_object()) { - Handle<WasmMemoryObject> memory_object = WasmMemoryObject::New( - isolate_, - instance->has_memory_buffer() ? handle(instance->memory_buffer()) - : Handle<JSArrayBuffer>::null(), - module_->maximum_pages != 0 ? module_->maximum_pages : -1); - instance->set_memory_object(*memory_object); - } - - code_specialization.RelocateWasmContextReferences( - reinterpret_cast<Address>(instance->wasm_context())); - // Store the wasm_context address in the JSToWasmWrapperCache so that it can - // be used to compile JSToWasmWrappers. - js_to_wasm_cache_.SetContextAddress( - reinterpret_cast<Address>(instance->wasm_context())); - } + // Set the WasmContext address in wrappers. + // TODO(wasm): the wasm context should only appear as a constant in wrappers; + // this code specialization is applied to the whole instance. + Address wasm_context_address = reinterpret_cast<Address>(wasm_context); + code_specialization.RelocateWasmContextReferences(wasm_context_address); + js_to_wasm_cache_.SetContextAddress(wasm_context_address); + + if (!FLAG_wasm_jit_to_native) { + //-------------------------------------------------------------------------- + // Set up the runtime support for the new instance. + //-------------------------------------------------------------------------- + Handle<WeakCell> weak_link = factory->NewWeakCell(instance); - //-------------------------------------------------------------------------- - // Set up the runtime support for the new instance. - //-------------------------------------------------------------------------- - Handle<WeakCell> weak_link = factory->NewWeakCell(instance); - - for (int i = num_imported_functions + FLAG_skip_compiling_wasm_funcs, - num_functions = static_cast<int>(module_->functions.size()); - i < num_functions; ++i) { - Handle<Code> code = handle(Code::cast(code_table->get(i)), isolate_); - if (code->kind() == Code::WASM_FUNCTION) { - Handle<FixedArray> deopt_data = factory->NewFixedArray(2, TENURED); - deopt_data->set(0, *weak_link); - deopt_data->set(1, Smi::FromInt(i)); - code->set_deoptimization_data(*deopt_data); - continue; - } - DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index()); - int deopt_len = code->deoptimization_data()->length(); - if (deopt_len == 0) continue; - DCHECK_LE(2, deopt_len); - DCHECK_EQ(i, Smi::ToInt(code->deoptimization_data()->get(1))); - code->deoptimization_data()->set(0, *weak_link); - // Entries [2, deopt_len) encode information about table exports of this - // function. This is rebuilt in {LoadTableSegments}, so reset it here. - for (int i = 2; i < deopt_len; ++i) { - code->deoptimization_data()->set_undefined(isolate_, i); + for (int i = num_imported_functions + FLAG_skip_compiling_wasm_funcs, + num_functions = static_cast<int>(module_->functions.size()); + i < num_functions; ++i) { + Handle<Code> code = handle(Code::cast(code_table->get(i)), isolate_); + if (code->kind() == Code::WASM_FUNCTION) { + AttachWasmFunctionInfo(isolate_, code, weak_link, i); + continue; + } + DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index()); + int deopt_len = code->deoptimization_data()->length(); + if (deopt_len == 0) continue; + DCHECK_LE(2, deopt_len); + DCHECK_EQ(i, Smi::ToInt(code->deoptimization_data()->get(1))); + code->deoptimization_data()->set(0, *weak_link); + // Entries [2, deopt_len) encode information about table exports of this + // function. This is rebuilt in {LoadTableSegments}, so reset it here. + for (int i = 2; i < deopt_len; ++i) { + code->deoptimization_data()->set_undefined(isolate_, i); + } } } @@ -1908,67 +2450,58 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { if (thrower_->error()) return {}; //-------------------------------------------------------------------------- - // Add instance to Memory object + // Initialize the indirect function tables. //-------------------------------------------------------------------------- - if (instance->has_memory_object()) { - Handle<WasmMemoryObject> memory(instance->memory_object(), isolate_); - WasmMemoryObject::AddInstance(isolate_, memory, instance); + if (function_table_count > 0) { + LoadTableSegments(code_table, instance); } //-------------------------------------------------------------------------- - // Initialize the indirect function tables. + // Initialize the memory by loading data segments. //-------------------------------------------------------------------------- - if (function_table_count > 0) LoadTableSegments(code_table, instance); + if (module_->data_segments.size() > 0) { + LoadDataSegments(wasm_context); + } // Patch all code with the relocations registered in code_specialization. code_specialization.RelocateDirectCalls(instance); code_specialization.ApplyToWholeInstance(*instance, SKIP_ICACHE_FLUSH); - FlushICache(isolate_, code_table); + if (FLAG_wasm_jit_to_native) { + FlushICache(isolate_, native_module); + } else { + FlushICache(isolate_, code_table); + } FlushICache(isolate_, wrapper_table); //-------------------------------------------------------------------------- // Unpack and notify signal handler of protected instructions. //-------------------------------------------------------------------------- if (trap_handler::UseTrapHandler()) { - UnpackAndRegisterProtectedInstructions(isolate_, code_table); + if (FLAG_wasm_jit_to_native) { + UnpackAndRegisterProtectedInstructions(isolate_, native_module); + } else { + UnpackAndRegisterProtectedInstructionsGC(isolate_, code_table); + } } //-------------------------------------------------------------------------- - // Set up and link the new instance. + // Insert the compiled module into the weak list of compiled modules. //-------------------------------------------------------------------------- { Handle<Object> global_handle = isolate_->global_handles()->Create(*instance); - Handle<WeakCell> link_to_clone = factory->NewWeakCell(compiled_module_); Handle<WeakCell> link_to_owning_instance = factory->NewWeakCell(instance); - MaybeHandle<WeakCell> link_to_original; - MaybeHandle<WasmCompiledModule> original; if (!owner.is_null()) { - // prepare the data needed for publishing in a chain, but don't link - // just yet, because - // we want all the publishing to happen free from GC interruptions, and - // so we do it in - // one GC-free scope afterwards. - original = handle(owner.ToHandleChecked()->compiled_module()); - link_to_original = factory->NewWeakCell(original.ToHandleChecked()); - } - // Publish the new instance to the instances chain. - { + // Publish the new instance to the instances chain. DisallowHeapAllocation no_gc; - if (!link_to_original.is_null()) { - compiled_module_->set_weak_next_instance( - link_to_original.ToHandleChecked()); - original.ToHandleChecked()->set_weak_prev_instance(link_to_clone); - compiled_module_->set_weak_wasm_module( - original.ToHandleChecked()->weak_wasm_module()); - } - module_object_->set_compiled_module(*compiled_module_); - compiled_module_->set_weak_owning_instance(link_to_owning_instance); - GlobalHandles::MakeWeak( - global_handle.location(), global_handle.location(), - instance_finalizer_callback_, v8::WeakCallbackType::kFinalizer); + compiled_module_->InsertInChain(*module_object_); } + module_object_->set_compiled_module(*compiled_module_); + compiled_module_->set_weak_owning_instance(link_to_owning_instance); + GlobalHandles::MakeWeak(global_handle.location(), global_handle.location(), + instance_finalizer_callback_, + v8::WeakCallbackType::kFinalizer); } //-------------------------------------------------------------------------- @@ -1993,22 +2526,24 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { } //-------------------------------------------------------------------------- - // Run the start function if one was specified. + // Execute the start function if one was specified. //-------------------------------------------------------------------------- if (module_->start_function_index >= 0) { HandleScope scope(isolate_); int start_index = module_->start_function_index; - Handle<Code> startup_code = EnsureExportedLazyDeoptData( - isolate_, instance, code_table, start_index); + WasmCodeWrapper startup_code = EnsureExportedLazyDeoptData( + isolate_, instance, code_table, native_module, start_index); FunctionSig* sig = module_->functions[start_index].sig; Handle<Code> wrapper_code = js_to_wasm_cache_.CloneOrCompileJSToWasmWrapper( isolate_, module_, startup_code, start_index); Handle<WasmExportedFunction> startup_fct = WasmExportedFunction::New( isolate_, instance, MaybeHandle<String>(), start_index, static_cast<int>(sig->parameter_count()), wrapper_code); - RecordStats(*startup_code, counters()); + RecordStats(startup_code, counters()); // Call the JS function. Handle<Object> undefined = factory->undefined_value(); + // Close the CodeSpaceMemoryModificationScope to execute the start function. + modification_scope.reset(); { // We're OK with JS execution here. The instance is fully setup. AllowJavascriptExecution allow_js(isolate_); @@ -2027,7 +2562,12 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() { } DCHECK(!isolate_->has_pending_exception()); - TRACE("Finishing instance %d\n", compiled_module_->instance_id()); + if (FLAG_wasm_jit_to_native) { + TRACE("Successfully built instance %zu\n", + compiled_module_->GetNativeModule()->instance_id); + } else { + TRACE("Finishing instance %d\n", compiled_module_->instance_id()); + } TRACE_CHAIN(module_object_->compiled_module()); return instance; } @@ -2117,7 +2657,7 @@ uint32_t InstanceBuilder::EvalUint32InitExpr(const WasmInitExpr& expr) { } // Load data segments into the memory. -void InstanceBuilder::LoadDataSegments(Address mem_addr, size_t mem_size) { +void InstanceBuilder::LoadDataSegments(WasmContext* wasm_context) { Handle<SeqOneByteString> module_bytes(compiled_module_->module_bytes(), isolate_); for (const WasmDataSegment& segment : module_->data_segments) { @@ -2125,9 +2665,8 @@ void InstanceBuilder::LoadDataSegments(Address mem_addr, size_t mem_size) { // Segments of size == 0 are just nops. if (source_size == 0) continue; uint32_t dest_offset = EvalUint32InitExpr(segment.dest_addr); - DCHECK( - in_bounds(dest_offset, source_size, static_cast<uint32_t>(mem_size))); - byte* dest = mem_addr + dest_offset; + DCHECK(in_bounds(dest_offset, source_size, wasm_context->mem_size)); + byte* dest = wasm_context->mem_start + dest_offset; const byte* src = reinterpret_cast<const byte*>( module_bytes->GetCharsAddress() + segment.source.offset()); memcpy(dest, src, source_size); @@ -2137,8 +2676,9 @@ void InstanceBuilder::LoadDataSegments(Address mem_addr, size_t mem_size) { void InstanceBuilder::WriteGlobalValue(WasmGlobal& global, Handle<Object> value) { double num = value->Number(); - TRACE("init [globals+%u] = %lf, type = %s\n", global.offset, num, - WasmOpcodes::TypeName(global.type)); + TRACE("init [globals_start=%p + %u] = %lf, type = %s\n", + reinterpret_cast<void*>(raw_buffer_ptr(globals_, 0)), global.offset, + num, WasmOpcodes::TypeName(global.type)); switch (global.type) { case kWasmI32: *GetRawGlobalPtr<int32_t>(global) = static_cast<int32_t>(num); @@ -2245,18 +2785,20 @@ int InstanceBuilder::ProcessImports(Handle<FixedArray> code_table, module_name, import_name); return -1; } - - Handle<Code> import_code = UnwrapExportOrCompileImportWrapper( - isolate_, index, module_->functions[import.index].sig, - Handle<JSReceiver>::cast(value), module_->origin(), - &imported_wasm_instances, js_imports_table, instance); + WasmCodeWrapper import_code = UnwrapExportOrCompileImportWrapper( + isolate_, module_->functions[import.index].sig, + Handle<JSReceiver>::cast(value), num_imported_functions, + module_->origin(), &imported_wasm_instances, js_imports_table, + instance); if (import_code.is_null()) { ReportLinkError("imported function does not match the expected type", index, module_name, import_name); return -1; } - code_table->set(num_imported_functions, *import_code); - RecordStats(*import_code, counters()); + if (!FLAG_wasm_jit_to_native) { + code_table->set(num_imported_functions, *import_code.GetCode()); + } + RecordStats(import_code, counters()); num_imported_functions++; break; } @@ -2270,6 +2812,7 @@ int InstanceBuilder::ProcessImports(Handle<FixedArray> code_table, module_->function_tables[num_imported_tables]; TableInstance& table_instance = table_instances_[num_imported_tables]; table_instance.table_object = Handle<WasmTableObject>::cast(value); + instance->set_table_object(*table_instance.table_object); table_instance.js_wrappers = Handle<FixedArray>( table_instance.table_object->functions(), isolate_); @@ -2313,17 +2856,55 @@ int InstanceBuilder::ProcessImports(Handle<FixedArray> code_table, // that are already in the table. for (int i = 0; i < table_size; ++i) { Handle<Object> val(table_instance.js_wrappers->get(i), isolate_); + // TODO(mtrofin): this is the same logic as WasmTableObject::Set: + // insert in the local table a wrapper from the other module, and add + // a reference to the owning instance of the other module. if (!val->IsJSFunction()) continue; - WasmFunction* function = GetWasmFunctionForExport(isolate_, val); - if (function == nullptr) { + if (!WasmExportedFunction::IsWasmExportedFunction(*val)) { thrower_->LinkError("table import %d[%d] is not a wasm function", index, i); return -1; } - int sig_index = table.map.FindOrInsert(function->sig); - table_instance.signature_table->set(i, Smi::FromInt(sig_index)); - table_instance.function_table->set( - i, *UnwrapExportWrapper(Handle<JSFunction>::cast(val))); + // Look up the signature's canonical id. If there is no canonical + // id, then the signature does not appear at all in this module, + // so putting {-1} in the table will cause checks to always fail. + auto target = Handle<WasmExportedFunction>::cast(val); + if (!FLAG_wasm_jit_to_native) { + FunctionSig* sig = nullptr; + Handle<Code> code = + MakeWasmToWasmWrapper(isolate_, target, nullptr, &sig, + &imported_wasm_instances, instance, 0) + .GetCode(); + int sig_index = module_->signature_map.Find(sig); + table_instance.signature_table->set(i, Smi::FromInt(sig_index)); + table_instance.function_table->set(i, *code); + } else { + const wasm::WasmCode* exported_code = + target->GetWasmCode().GetWasmCode(); + wasm::NativeModule* exporting_module = exported_code->owner(); + Handle<WasmInstanceObject> imported_instance = + handle(target->instance()); + imported_wasm_instances.Set(imported_instance, imported_instance); + FunctionSig* sig = imported_instance->module() + ->functions[exported_code->index()] + .sig; + wasm::WasmCode* wrapper_code = + exporting_module->GetExportedWrapper(exported_code->index()); + if (wrapper_code == nullptr) { + WasmContext* other_context = + imported_instance->wasm_context()->get(); + Handle<Code> wrapper = compiler::CompileWasmToWasmWrapper( + isolate_, target->GetWasmCode(), sig, + reinterpret_cast<Address>(other_context)); + wrapper_code = exporting_module->AddExportedWrapper( + wrapper, exported_code->index()); + } + int sig_index = module_->signature_map.Find(sig); + table_instance.signature_table->set(i, Smi::FromInt(sig_index)); + Handle<Foreign> foreign_holder = isolate_->factory()->NewForeign( + wrapper_code->instructions().start(), TENURED); + table_instance.function_table->set(i, *foreign_holder); + } } num_imported_tables++; @@ -2642,7 +3223,7 @@ void InstanceBuilder::ProcessExports( } v8::Maybe<bool> status = JSReceiver::DefineOwnProperty( - isolate_, export_to, name, &desc, Object::THROW_ON_ERROR); + isolate_, export_to, name, &desc, kThrowOnError); if (!status.IsJust()) { TruncatedUserString<> trunc_name(name->GetCharVector<uint8_t>()); thrower_->LinkError("export of %.*s failed.", trunc_name.length(), @@ -2653,8 +3234,8 @@ void InstanceBuilder::ProcessExports( DCHECK_EQ(export_index, weak_exported_functions->length()); if (module_->is_wasm()) { - v8::Maybe<bool> success = JSReceiver::SetIntegrityLevel( - exports_object, FROZEN, Object::DONT_THROW); + v8::Maybe<bool> success = + JSReceiver::SetIntegrityLevel(exports_object, FROZEN, kDontThrow); DCHECK(success.FromMaybe(false)); USE(success); } @@ -2663,28 +3244,56 @@ void InstanceBuilder::ProcessExports( void InstanceBuilder::InitializeTables( Handle<WasmInstanceObject> instance, CodeSpecialization* code_specialization) { - int function_table_count = static_cast<int>(module_->function_tables.size()); - Handle<FixedArray> new_function_tables = - isolate_->factory()->NewFixedArray(function_table_count, TENURED); - Handle<FixedArray> new_signature_tables = - isolate_->factory()->NewFixedArray(function_table_count, TENURED); - Handle<FixedArray> old_function_tables = compiled_module_->function_tables(); - Handle<FixedArray> old_signature_tables = - compiled_module_->signature_tables(); + size_t function_table_count = module_->function_tables.size(); + std::vector<GlobalHandleAddress> new_function_tables(function_table_count); + std::vector<GlobalHandleAddress> new_signature_tables(function_table_count); + + wasm::NativeModule* native_module = compiled_module_->GetNativeModule(); + std::vector<GlobalHandleAddress> empty; + std::vector<GlobalHandleAddress>& old_function_tables = + FLAG_wasm_jit_to_native ? native_module->function_tables() : empty; + std::vector<GlobalHandleAddress>& old_signature_tables = + FLAG_wasm_jit_to_native ? native_module->signature_tables() : empty; + + Handle<FixedArray> old_function_tables_gc = + FLAG_wasm_jit_to_native ? Handle<FixedArray>::null() + : compiled_module_->function_tables(); + Handle<FixedArray> old_signature_tables_gc = + FLAG_wasm_jit_to_native ? Handle<FixedArray>::null() + : compiled_module_->signature_tables(); + + // function_table_count is 0 or 1, so we just create these objects even if not + // needed for native wasm. + // TODO(mtrofin): remove the {..}_gc variables when we don't need + // FLAG_wasm_jit_to_native + Handle<FixedArray> new_function_tables_gc = + isolate_->factory()->NewFixedArray(static_cast<int>(function_table_count), + TENURED); + Handle<FixedArray> new_signature_tables_gc = + isolate_->factory()->NewFixedArray(static_cast<int>(function_table_count), + TENURED); // These go on the instance. Handle<FixedArray> rooted_function_tables = - isolate_->factory()->NewFixedArray(function_table_count, TENURED); + isolate_->factory()->NewFixedArray(static_cast<int>(function_table_count), + TENURED); Handle<FixedArray> rooted_signature_tables = - isolate_->factory()->NewFixedArray(function_table_count, TENURED); + isolate_->factory()->NewFixedArray(static_cast<int>(function_table_count), + TENURED); instance->set_function_tables(*rooted_function_tables); instance->set_signature_tables(*rooted_signature_tables); - DCHECK_EQ(old_function_tables->length(), new_function_tables->length()); - DCHECK_EQ(old_signature_tables->length(), new_signature_tables->length()); - - for (int index = 0; index < function_table_count; ++index) { + if (FLAG_wasm_jit_to_native) { + DCHECK_EQ(old_function_tables.size(), new_function_tables.size()); + DCHECK_EQ(old_signature_tables.size(), new_signature_tables.size()); + } else { + DCHECK_EQ(old_function_tables_gc->length(), + new_function_tables_gc->length()); + DCHECK_EQ(old_signature_tables_gc->length(), + new_signature_tables_gc->length()); + } + for (size_t index = 0; index < function_table_count; ++index) { WasmIndirectFunctionTable& table = module_->function_tables[index]; TableInstance& table_instance = table_instances_[index]; int table_size = static_cast<int>(table.initial_size); @@ -2734,31 +3343,45 @@ void InstanceBuilder::InitializeTables( GlobalHandleAddress new_func_table_addr = global_func_table.address(); GlobalHandleAddress new_sig_table_addr = global_sig_table.address(); - WasmCompiledModule::SetTableValue(isolate_, new_function_tables, int_index, - new_func_table_addr); - WasmCompiledModule::SetTableValue(isolate_, new_signature_tables, int_index, - new_sig_table_addr); - - GlobalHandleAddress old_func_table_addr = - WasmCompiledModule::GetTableValue(*old_function_tables, int_index); - GlobalHandleAddress old_sig_table_addr = - WasmCompiledModule::GetTableValue(*old_signature_tables, int_index); + GlobalHandleAddress old_func_table_addr; + GlobalHandleAddress old_sig_table_addr; + if (!FLAG_wasm_jit_to_native) { + WasmCompiledModule::SetTableValue(isolate_, new_function_tables_gc, + int_index, new_func_table_addr); + WasmCompiledModule::SetTableValue(isolate_, new_signature_tables_gc, + int_index, new_sig_table_addr); + + old_func_table_addr = + WasmCompiledModule::GetTableValue(*old_function_tables_gc, int_index); + old_sig_table_addr = WasmCompiledModule::GetTableValue( + *old_signature_tables_gc, int_index); + } else { + new_function_tables[int_index] = new_func_table_addr; + new_signature_tables[int_index] = new_sig_table_addr; + old_func_table_addr = old_function_tables[int_index]; + old_sig_table_addr = old_signature_tables[int_index]; + } code_specialization->RelocatePointer(old_func_table_addr, new_func_table_addr); code_specialization->RelocatePointer(old_sig_table_addr, new_sig_table_addr); } - compiled_module_->set_function_tables(new_function_tables); - compiled_module_->set_signature_tables(new_signature_tables); + if (FLAG_wasm_jit_to_native) { + native_module->function_tables() = new_function_tables; + native_module->signature_tables() = new_signature_tables; + } else { + compiled_module_->set_function_tables(new_function_tables_gc); + compiled_module_->set_signature_tables(new_signature_tables_gc); + } } void InstanceBuilder::LoadTableSegments(Handle<FixedArray> code_table, Handle<WasmInstanceObject> instance) { + wasm::NativeModule* native_module = compiled_module_->GetNativeModule(); int function_table_count = static_cast<int>(module_->function_tables.size()); for (int index = 0; index < function_table_count; ++index) { - WasmIndirectFunctionTable& table = module_->function_tables[index]; TableInstance& table_instance = table_instances_[index]; Handle<FixedArray> all_dispatch_tables; @@ -2774,12 +3397,20 @@ void InstanceBuilder::LoadTableSegments(Handle<FixedArray> code_table, if (compile_lazy(module_)) { for (auto& table_init : module_->table_inits) { for (uint32_t func_index : table_init.entries) { - Code* code = - Code::cast(code_table->get(static_cast<int>(func_index))); - // Only increase the counter for lazy compile builtins (it's not - // needed otherwise). - if (code->is_wasm_code()) continue; - DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index()); + if (!FLAG_wasm_jit_to_native) { + Code* code = + Code::cast(code_table->get(static_cast<int>(func_index))); + // Only increase the counter for lazy compile builtins (it's not + // needed otherwise). + if (code->is_wasm_code()) continue; + DCHECK_EQ(Builtins::kWasmCompileLazy, code->builtin_index()); + } else { + const wasm::WasmCode* code = native_module->GetCode(func_index); + // Only increase the counter for lazy compile builtins (it's not + // needed otherwise). + if (code->kind() == wasm::WasmCode::Function) continue; + DCHECK_EQ(wasm::WasmCode::LazyStub, code->kind()); + } ++num_table_exports[func_index]; } } @@ -2796,15 +3427,22 @@ void InstanceBuilder::LoadTableSegments(Handle<FixedArray> code_table, uint32_t func_index = table_init.entries[i]; WasmFunction* function = &module_->functions[func_index]; int table_index = static_cast<int>(i + base); - int32_t sig_index = table.map.Find(function->sig); - DCHECK_GE(sig_index, 0); + uint32_t sig_index = module_->signature_ids[function->sig_index]; table_instance.signature_table->set(table_index, Smi::FromInt(sig_index)); - Handle<Code> wasm_code = EnsureTableExportLazyDeoptData( - isolate_, instance, code_table, func_index, - table_instance.function_table, table_index, num_table_exports); - table_instance.function_table->set(table_index, *wasm_code); - + WasmCodeWrapper wasm_code = EnsureTableExportLazyDeoptData( + isolate_, instance, code_table, native_module, func_index, + table_instance.function_table, table_index, &num_table_exports); + Handle<Object> value_to_update_with; + if (!wasm_code.IsCodeObject()) { + Handle<Foreign> as_foreign = isolate_->factory()->NewForeign( + wasm_code.GetWasmCode()->instructions().start(), TENURED); + table_instance.function_table->set(table_index, *as_foreign); + value_to_update_with = as_foreign; + } else { + table_instance.function_table->set(table_index, *wasm_code.GetCode()); + value_to_update_with = wasm_code.GetCode(); + } if (!all_dispatch_tables.is_null()) { if (js_wrappers_[func_index].is_null()) { // No JSFunction entry yet exists for this function. Create one. @@ -2831,9 +3469,30 @@ void InstanceBuilder::LoadTableSegments(Handle<FixedArray> code_table, } table_instance.js_wrappers->set(table_index, *js_wrappers_[func_index]); - + // When updating dispatch tables, we need to provide a wasm-to-wasm + // wrapper for wasm_code - unless wasm_code is already a wrapper. If + // it's a wasm-to-js wrapper, we don't need to construct a + // wasm-to-wasm wrapper because there's no context switching required. + // The remaining case is that it's a wasm-to-wasm wrapper, in which + // case it's already doing "the right thing", and wrapping it again + // would be redundant. + if (func_index >= module_->num_imported_functions) { + value_to_update_with = GetOrCreateIndirectCallWrapper( + isolate_, instance, wasm_code, func_index, function->sig); + } else { + if (wasm_code.IsCodeObject()) { + DCHECK(wasm_code.GetCode()->kind() == Code::WASM_TO_JS_FUNCTION || + wasm_code.GetCode()->kind() == + Code::WASM_TO_WASM_FUNCTION); + } else { + DCHECK(wasm_code.GetWasmCode()->kind() == + WasmCode::WasmToJsWrapper || + wasm_code.GetWasmCode()->kind() == + WasmCode::WasmToWasmWrapper); + } + } UpdateDispatchTables(isolate_, all_dispatch_tables, table_index, - function, wasm_code); + function, value_to_update_with); } } } @@ -2866,6 +3525,10 @@ AsyncCompileJob::AsyncCompileJob(Isolate* isolate, async_counters_(isolate->async_counters()), bytes_copy_(std::move(bytes_copy)), wire_bytes_(bytes_copy_.get(), bytes_copy_.get() + length) { + v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate); + v8::Platform* platform = V8::GetCurrentPlatform(); + foreground_task_runner_ = platform->GetForegroundTaskRunner(v8_isolate); + background_task_runner_ = platform->GetBackgroundTaskRunner(v8_isolate); // The handles for the context and promise must be deferred. DeferredHandleScope deferred(isolate); context_ = Handle<Context>(*context); @@ -3007,11 +3670,7 @@ void AsyncCompileJob::StartForegroundTask() { ++num_pending_foreground_tasks_; DCHECK_EQ(1, num_pending_foreground_tasks_); - v8::Platform* platform = V8::GetCurrentPlatform(); - // TODO(ahaas): This is a CHECK to debug issue 764313. - CHECK(platform); - platform->CallOnForegroundThread(reinterpret_cast<v8::Isolate*>(isolate_), - new CompileTask(this, true)); + foreground_task_runner_->PostTask(base::make_unique<CompileTask>(this, true)); } template <typename Step, typename... Args> @@ -3021,8 +3680,8 @@ void AsyncCompileJob::DoSync(Args&&... args) { } void AsyncCompileJob::StartBackgroundTask() { - V8::GetCurrentPlatform()->CallOnBackgroundThread( - new CompileTask(this, false), v8::Platform::kShortRunningTask); + background_task_runner_->PostTask( + base::make_unique<CompileTask>(this, false)); } void AsyncCompileJob::RestartBackgroundTasks() { @@ -3111,26 +3770,30 @@ class AsyncCompileJob::PrepareAndStartCompile : public CompileStep { void RunInForeground() override { TRACE_COMPILE("(2) Prepare and start compile...\n"); Isolate* isolate = job_->isolate_; - Factory* factory = isolate->factory(); + Handle<Code> illegal_builtin = BUILTIN_CODE(isolate, Illegal); + if (!FLAG_wasm_jit_to_native) { + // The {code_table} array contains import wrappers and functions (which + // are both included in {functions.size()}. + // The results of compilation will be written into it. + // Initialize {code_table_} with the illegal builtin. All call sites + // will be patched at instantiation. + int code_table_size = static_cast<int>(module_->functions.size()); + job_->code_table_ = factory->NewFixedArray(code_table_size, TENURED); + + for (int i = 0, e = module_->num_imported_functions; i < e; ++i) { + job_->code_table_->set(i, *illegal_builtin); + } + } else { + // Just makes it easier to deal with code that wants code_table, while + // we have FLAG_wasm_jit_to_native around. + job_->code_table_ = factory->NewFixedArray(0, TENURED); + } + job_->module_env_ = CreateDefaultModuleEnv(isolate, module_, illegal_builtin); - // The {code_table} array contains import wrappers and functions (which - // are both included in {functions.size()}. - // The results of compilation will be written into it. - // Initialize {code_table_} with the illegal builtin. All call sites - // will be patched at instantiation. - int code_table_size = static_cast<int>(module_->functions.size()); - int export_wrapper_size = static_cast<int>(module_->num_exported_functions); - job_->code_table_ = factory->NewFixedArray(code_table_size, TENURED); - job_->export_wrappers_ = - factory->NewFixedArray(export_wrapper_size, TENURED); - - for (int i = 0, e = module_->num_imported_functions; i < e; ++i) { - job_->code_table_->set(i, *illegal_builtin); - } // Transfer ownership of the {WasmModule} to the {ModuleCompiler}, but // keep a pointer. Handle<Code> centry_stub = CEntryStub(isolate, 1).GetCode(); @@ -3141,8 +3804,6 @@ class AsyncCompileJob::PrepareAndStartCompile : public CompileStep { centry_stub = Handle<Code>(*centry_stub, isolate); job_->code_table_ = Handle<FixedArray>(*job_->code_table_, isolate); - job_->export_wrappers_ = - Handle<FixedArray>(*job_->export_wrappers_, isolate); compiler::ModuleEnv* env = job_->module_env_.get(); ReopenHandles(isolate, env->function_code); Handle<Code>* mut = @@ -3152,12 +3813,32 @@ class AsyncCompileJob::PrepareAndStartCompile : public CompileStep { job_->deferred_handles_.push_back(deferred.Detach()); } - job_->compiler_.reset(new ModuleCompiler(isolate, module_, centry_stub)); + DCHECK_LE(module_->num_imported_functions, module_->functions.size()); + // Create the compiled module object and populate with compiled functions + // and information needed at instantiation time. This object needs to be + // serializable. Instantiation may occur off a deserialized version of + // this object. + int export_wrapper_size = static_cast<int>(module_->num_exported_functions); + Handle<FixedArray> export_wrappers = + job_->isolate_->factory()->NewFixedArray(export_wrapper_size, TENURED); + + job_->compiled_module_ = + NewCompiledModule(job_->isolate_, module_, job_->code_table_, + export_wrappers, job_->module_env_.get()); + + job_->compiler_.reset( + new ModuleCompiler(isolate, module_, centry_stub, + job_->compiled_module_->GetNativeModule())); job_->compiler_->EnableThrottling(); - DCHECK_LE(module_->num_imported_functions, module_->functions.size()); + { + DeferredHandleScope deferred(job_->isolate_); + job_->compiled_module_ = handle(*job_->compiled_module_, job_->isolate_); + job_->deferred_handles_.push_back(deferred.Detach()); + } size_t num_functions = module_->functions.size() - module_->num_imported_functions; + if (num_functions == 0) { // Degenerate case of an empty module. job_->DoSync<FinishCompile>(); @@ -3171,7 +3852,6 @@ class AsyncCompileJob::PrepareAndStartCompile : public CompileStep { Min(static_cast<size_t>(FLAG_wasm_num_compilation_tasks), V8::GetCurrentPlatform() ->NumberOfAvailableBackgroundThreads()))); - if (start_compilation_) { // TODO(ahaas): Try to remove the {start_compilation_} check when // streaming decoding is done in the background. If @@ -3216,6 +3896,11 @@ class AsyncCompileJob::ExecuteAndFinishCompilationUnits : public CompileStep { } void RunInForeground() override { + // TODO(6792): No longer needed once WebAssembly code is off heap. + // Use base::Optional to be able to close the scope before we resolve or + // reject the promise. + base::Optional<CodeSpaceMemoryModificationScope> modification_scope( + base::in_place_t(), job_->isolate_->heap()); TRACE_COMPILE("(4a) Finishing compilation units...\n"); if (failed_) { // The job failed already, no need to do more work. @@ -3234,7 +3919,7 @@ class AsyncCompileJob::ExecuteAndFinishCompilationUnits : public CompileStep { int func_index = -1; - MaybeHandle<Code> result = + WasmCodeWrapper result = job_->compiler_->FinishCompilationUnit(&thrower, &func_index); if (thrower.error()) { @@ -3249,7 +3934,9 @@ class AsyncCompileJob::ExecuteAndFinishCompilationUnits : public CompileStep { break; } else { DCHECK_LE(0, func_index); - job_->code_table_->set(func_index, *result.ToHandleChecked()); + if (result.IsCodeObject()) { + job_->code_table_->set(func_index, *result.GetCode()); + } --job_->outstanding_units_; } @@ -3267,6 +3954,10 @@ class AsyncCompileJob::ExecuteAndFinishCompilationUnits : public CompileStep { if (thrower.error()) { // Make sure all compilation tasks stopped running. job_->background_task_manager_.CancelAndWait(); + + // Close the CodeSpaceMemoryModificationScope before we reject the promise + // in AsyncCompileFailed. Promise::Reject calls directly into JavaScript. + modification_scope.reset(); return job_->AsyncCompileFailed(thrower); } if (job_->outstanding_units_ == 0) { @@ -3287,12 +3978,16 @@ class AsyncCompileJob::ExecuteAndFinishCompilationUnits : public CompileStep { class AsyncCompileJob::FinishCompile : public CompileStep { void RunInForeground() override { TRACE_COMPILE("(5b) Finish compile...\n"); - // At this point, compilation has completed. Update the code table. - for (int i = FLAG_skip_compiling_wasm_funcs, - e = job_->code_table_->length(); - i < e; ++i) { - Object* val = job_->code_table_->get(i); - if (val->IsCode()) RecordStats(Code::cast(val), job_->counters()); + if (FLAG_wasm_jit_to_native) { + RecordStats(job_->compiled_module_->GetNativeModule(), job_->counters()); + } else { + // At this point, compilation has completed. Update the code table. + for (int i = FLAG_skip_compiling_wasm_funcs, + e = job_->code_table_->length(); + i < e; ++i) { + Object* val = job_->code_table_->get(i); + if (val->IsCode()) RecordStats(Code::cast(val), job_->counters()); + } } // Create heap objects for script and module bytes to be stored in the @@ -3326,21 +4021,13 @@ class AsyncCompileJob::FinishCompile : public CompileStep { WasmSharedModuleData::New(job_->isolate_, module_wrapper, Handle<SeqOneByteString>::cast(module_bytes), script, asm_js_offset_table); + job_->compiled_module_->OnWasmModuleDecodingComplete(shared); + script->set_wasm_compiled_module(*job_->compiled_module_); - // Create the compiled module object and populate with compiled functions - // and information needed at instantiation time. This object needs to be - // serializable. Instantiation may occur off a deserialized version of - // this object. - job_->compiled_module_ = - NewCompiledModule(job_->isolate_, shared, job_->code_table_, - job_->export_wrappers_, job_->module_env_.get()); // Finish the wasm script now and make it public to the debugger. - script->set_wasm_compiled_module(*job_->compiled_module_); - job_->isolate_->debug()->OnAfterCompile(script); + job_->isolate_->debug()->OnAfterCompile( + handle(job_->compiled_module_->script())); - DeferredHandleScope deferred(job_->isolate_); - job_->compiled_module_ = handle(*job_->compiled_module_, job_->isolate_); - job_->deferred_handles_.push_back(deferred.Detach()); // TODO(wasm): compiling wrappers should be made async as well. job_->DoSync<CompileWrappers>(); } @@ -3354,22 +4041,11 @@ class AsyncCompileJob::CompileWrappers : public CompileStep { // and the wrappers for the function table elements. void RunInForeground() override { TRACE_COMPILE("(6) Compile wrappers...\n"); + // TODO(6792): No longer needed once WebAssembly code is off heap. + CodeSpaceMemoryModificationScope modification_scope(job_->isolate_->heap()); // Compile JS->wasm wrappers for exported functions. - JSToWasmWrapperCache js_to_wasm_cache; - int wrapper_index = 0; - WasmModule* module = job_->compiled_module_->module(); - for (auto exp : module->export_table) { - if (exp.kind != kExternalFunction) continue; - Handle<Code> wasm_code(Code::cast(job_->code_table_->get(exp.index)), - job_->isolate_); - Handle<Code> wrapper_code = - js_to_wasm_cache.CloneOrCompileJSToWasmWrapper(job_->isolate_, module, - wasm_code, exp.index); - job_->export_wrappers_->set(wrapper_index, *wrapper_code); - RecordStats(*wrapper_code, job_->counters()); - ++wrapper_index; - } - + CompileJsToWasmWrappers(job_->isolate_, job_->compiled_module_, + job_->counters()); job_->DoSync<FinishModule>(); } }; @@ -3483,8 +4159,6 @@ bool AsyncStreamingProcessor::ProcessCodeSectionHeader(size_t functions_count, // Set outstanding_finishers_ to 2, because both the AsyncCompileJob and the // AsyncStreamingProcessor have to finish. job_->outstanding_finishers_.SetValue(2); - next_function_ = decoder_.module()->num_imported_functions + - FLAG_skip_compiling_wasm_funcs; compilation_unit_builder_.reset( new ModuleCompiler::CompilationUnitBuilder(job_->compiler_.get())); return true; @@ -3495,16 +4169,20 @@ bool AsyncStreamingProcessor::ProcessFunctionBody(Vector<const uint8_t> bytes, uint32_t offset) { TRACE_STREAMING("Process function body %d ...\n", next_function_); - decoder_.DecodeFunctionBody( - next_function_, static_cast<uint32_t>(bytes.length()), offset, false); - if (next_function_ >= decoder_.module()->num_imported_functions + - FLAG_skip_compiling_wasm_funcs) { - const WasmFunction* func = &decoder_.module()->functions[next_function_]; + if (next_function_ >= FLAG_skip_compiling_wasm_funcs) { + decoder_.DecodeFunctionBody( + next_function_, static_cast<uint32_t>(bytes.length()), offset, false); + + uint32_t index = next_function_ + decoder_.module()->num_imported_functions; + const WasmFunction* func = &decoder_.module()->functions[index]; WasmName name = {nullptr, 0}; - compilation_unit_builder_->AddUnit(job_->module_env_.get(), func, offset, - bytes, name); + compilation_unit_builder_->AddUnit( + job_->module_env_.get(), job_->compiled_module_->GetNativeModule(), + func, offset, bytes, name); } ++next_function_; + // This method always succeeds. The return value is necessary to comply with + // the StreamingProcessor interface. return true; } @@ -3526,8 +4204,18 @@ void AsyncStreamingProcessor::OnFinishedStream(std::unique_ptr<uint8_t[]> bytes, ModuleResult result = decoder_.FinishDecoding(false); DCHECK(result.ok()); job_->module_ = std::move(result.val); - if (job_->DecrementAndCheckFinisherCount()) - job_->DoSync<AsyncCompileJob::FinishCompile>(); + if (job_->DecrementAndCheckFinisherCount()) { + if (!job_->compiler_) { + // We are processing a WebAssembly module without code section. We need to + // prepare compilation first before we can finish it. + // {PrepareAndStartCompile} will call {FinishCompile} by itself if there + // is no code section. + job_->DoSync<AsyncCompileJob::PrepareAndStartCompile>(job_->module_.get(), + true); + } else { + job_->DoSync<AsyncCompileJob::FinishCompile>(); + } + } } // Report an error detected in the StreamingDecoder. @@ -3541,12 +4229,62 @@ void AsyncStreamingProcessor::OnAbort() { job_->Abort(); } +void CompileJsToWasmWrappers(Isolate* isolate, + Handle<WasmCompiledModule> compiled_module, + Counters* counters) { + JSToWasmWrapperCache js_to_wasm_cache; + int wrapper_index = 0; + Handle<FixedArray> export_wrappers = compiled_module->export_wrappers(); + NativeModule* native_module = compiled_module->GetNativeModule(); + for (auto exp : compiled_module->module()->export_table) { + if (exp.kind != kExternalFunction) continue; + WasmCodeWrapper wasm_code = EnsureExportedLazyDeoptData( + isolate, Handle<WasmInstanceObject>::null(), + compiled_module->code_table(), native_module, exp.index); + Handle<Code> wrapper_code = js_to_wasm_cache.CloneOrCompileJSToWasmWrapper( + isolate, compiled_module->module(), wasm_code, exp.index); + export_wrappers->set(wrapper_index, *wrapper_code); + RecordStats(*wrapper_code, counters); + ++wrapper_index; + } +} + +Handle<Script> CreateWasmScript(Isolate* isolate, + const ModuleWireBytes& wire_bytes) { + Handle<Script> script = + isolate->factory()->NewScript(isolate->factory()->empty_string()); + script->set_context_data(isolate->native_context()->debug_context_id()); + script->set_type(Script::TYPE_WASM); + + int hash = StringHasher::HashSequentialString( + reinterpret_cast<const char*>(wire_bytes.start()), + static_cast<int>(wire_bytes.length()), kZeroHashSeed); + + const int kBufferSize = 32; + char buffer[kBufferSize]; + int url_chars = SNPrintF(ArrayVector(buffer), "wasm://wasm/%08x", hash); + DCHECK(url_chars >= 0 && url_chars < kBufferSize); + MaybeHandle<String> url_str = isolate->factory()->NewStringFromOneByte( + Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), url_chars), + TENURED); + script->set_source_url(*url_str.ToHandleChecked()); + + int name_chars = SNPrintF(ArrayVector(buffer), "wasm-%08x", hash); + DCHECK(name_chars >= 0 && name_chars < kBufferSize); + MaybeHandle<String> name_str = isolate->factory()->NewStringFromOneByte( + Vector<const uint8_t>(reinterpret_cast<uint8_t*>(buffer), name_chars), + TENURED); + script->set_name(*name_str.ToHandleChecked()); + + return script; +} + } // namespace wasm } // namespace internal } // namespace v8 #undef TRACE +#undef TRACE_CHAIN #undef TRACE_COMPILE #undef TRACE_STREAMING -#undef TRACE_CHAIN -#undef ERROR_THROWER_WITH_MESSAGE +#undef TRACE_LAZY |