diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | node.gyp | 53 | ||||
| -rw-r--r-- | src/node_native_module_env.cc | 24 | ||||
| -rw-r--r-- | src/node_native_module_env.h | 1 | ||||
| -rw-r--r-- | test/code-cache/test-code-cache-generator.js | 34 | ||||
| -rw-r--r-- | tools/code_cache/cache_builder.cc | 165 | ||||
| -rw-r--r-- | tools/code_cache/cache_builder.h | 16 | ||||
| -rw-r--r-- | tools/code_cache/mkcodecache.cc | 62 | ||||
| -rw-r--r-- | tools/generate_code_cache.js | 135 |
9 files changed, 322 insertions, 172 deletions
@@ -115,7 +115,7 @@ with-code-cache: $(PYTHON) ./configure $(CONFIG_FLAGS) $(MAKE) mkdir -p $(CODE_CACHE_DIR) - out/$(BUILDTYPE)/$(NODE_EXE) --expose-internals tools/generate_code_cache.js $(CODE_CACHE_FILE) + out/$(BUILDTYPE)/mkcodecache $(CODE_CACHE_FILE) $(PYTHON) ./configure --code-cache-path $(CODE_CACHE_FILE) $(CONFIG_FLAGS) $(MAKE) @@ -1232,6 +1232,8 @@ LINT_CPP_FILES = $(filter-out $(LINT_CPP_EXCLUDE), $(wildcard \ test/node-api/*/*.h \ tools/icu/*.cc \ tools/icu/*.h \ + tools/code_cache/*.cc \ + tools/code_cache/*.h \ )) # Code blocks don't have newline at the end, @@ -1103,6 +1103,59 @@ }], ], }, # cctest + # TODO(joyeecheung): do not depend on node_lib, + # instead create a smaller static library node_lib_base that does + # just enough for node_native_module.cc and the cache builder to + # compile without compiling the generated code cache C++ file. + # So generate_code_cache -> mkcodecache -> node_lib_base, + # node_lib -> node_lib_base & generate_code_cache + { + 'target_name': 'mkcodecache', + 'type': 'executable', + + 'dependencies': [ + '<(node_lib_target_name)', + 'deps/histogram/histogram.gyp:histogram', + ], + + 'includes': [ + 'node.gypi' + ], + + 'include_dirs': [ + 'src', + 'tools/msvs/genfiles', + 'deps/v8/include', + 'deps/cares/include', + 'deps/uv/include', + ], + + 'defines': [ 'NODE_WANT_INTERNALS=1' ], + + 'sources': [ + 'tools/code_cache/mkcodecache.cc', + 'tools/code_cache/cache_builder.cc' + ], + + 'conditions': [ + [ 'node_report=="true"', { + 'conditions': [ + ['OS=="win"', { + 'libraries': [ + 'dbghelp.lib', + 'PsApi.lib', + 'Ws2_32.lib', + ], + 'dll_files': [ + 'dbghelp.dll', + 'PsApi.dll', + 'Ws2_32.dll', + ], + }], + ], + }], + ], + }, # cache_builder ], # end targets 'conditions': [ diff --git a/src/node_native_module_env.cc b/src/node_native_module_env.cc index fc48436dc1..171c649a19 100644 --- a/src/node_native_module_env.cc +++ b/src/node_native_module_env.cc @@ -4,7 +4,6 @@ namespace node { namespace native_module { -using v8::ArrayBuffer; using v8::Context; using v8::DEFAULT; using v8::Function; @@ -18,11 +17,9 @@ using v8::Name; using v8::None; using v8::Object; using v8::PropertyCallbackInfo; -using v8::ScriptCompiler; using v8::Set; using v8::SideEffectType; using v8::String; -using v8::Uint8Array; using v8::Value; // TODO(joyeecheung): make these more general and put them into util.h @@ -154,26 +151,6 @@ MaybeLocal<Function> NativeModuleEnv::LookupAndCompile( return maybe; } -// This is supposed to be run only by the main thread in -// tools/generate_code_cache.js -void NativeModuleEnv::GetCodeCache(const FunctionCallbackInfo<Value>& args) { - Environment* env = Environment::GetCurrent(args); - Isolate* isolate = env->isolate(); - CHECK(env->is_main_thread()); - - CHECK(args[0]->IsString()); - node::Utf8Value id_v(isolate, args[0].As<String>()); - const char* id = *id_v; - - ScriptCompiler::CachedData* cached_data = - NativeModuleLoader::GetInstance()->GetCodeCache(id); - if (cached_data != nullptr) { - Local<ArrayBuffer> buf = ArrayBuffer::New(isolate, cached_data->length); - memcpy(buf->GetContents().Data(), cached_data->data, cached_data->length); - args.GetReturnValue().Set(Uint8Array::New(buf, 0, cached_data->length)); - } -} - // TODO(joyeecheung): It is somewhat confusing that Class::Initialize // is used to initilaize to the binding, but it is the current convention. // Rename this across the code base to something that makes more sense. @@ -216,7 +193,6 @@ void NativeModuleEnv::Initialize(Local<Object> target, .Check(); env->SetMethod(target, "getCacheUsage", NativeModuleEnv::GetCacheUsage); - env->SetMethod(target, "getCodeCache", NativeModuleEnv::GetCodeCache); env->SetMethod(target, "compileFunction", NativeModuleEnv::CompileFunction); // internalBinding('native_module') should be frozen target->SetIntegrityLevel(context, IntegrityLevel::kFrozen).FromJust(); diff --git a/src/node_native_module_env.h b/src/node_native_module_env.h index 2b1fabf89b..1852d9b139 100644 --- a/src/node_native_module_env.h +++ b/src/node_native_module_env.h @@ -52,7 +52,6 @@ class NativeModuleEnv { const v8::PropertyCallbackInfo<v8::Value>& info); // Compile a specific native module as a function static void CompileFunction(const v8::FunctionCallbackInfo<v8::Value>& args); - static void GetCodeCache(const v8::FunctionCallbackInfo<v8::Value>& args); }; } // namespace native_module diff --git a/test/code-cache/test-code-cache-generator.js b/test/code-cache/test-code-cache-generator.js index a117a2e6a6..2fed3294bf 100644 --- a/test/code-cache/test-code-cache-generator.js +++ b/test/code-cache/test-code-cache-generator.js @@ -1,10 +1,10 @@ 'use strict'; -// This test verifies that the binary is compiled with code cache and the -// cache is used when built in modules are compiled. +// This test verifies the code cache generator can generate a C++ +// file that contains the code cache. This can be removed once we +// actually build that C++ file into our binary. const common = require('../common'); - const tmpdir = require('../common/tmpdir'); const { spawnSync } = require('child_process'); const assert = require('assert'); @@ -12,17 +12,29 @@ const path = require('path'); const fs = require('fs'); const readline = require('readline'); -const generator = path.join( - __dirname, '..', '..', 'tools', 'generate_code_cache.js' -); +console.log('Looking for mkcodecache executable'); +let buildDir; +const stat = fs.statSync(process.execPath); +if (stat.isSymbolicLink()) { + console.log('Binary is a symbolic link'); + buildDir = path.dirname(fs.readlinkSync(process.execPath)); +} else { + buildDir = path.dirname(process.execPath); +} + +const ext = common.isWindows ? '.exe' : ''; +const generator = path.join(buildDir, `mkcodecache${ext}`); +if (!fs.existsSync(generator)) { + common.skip('Could not find mkcodecache'); +} + +console.log(`mkcodecache is ${generator}`); + tmpdir.refresh(); const dest = path.join(tmpdir.path, 'cache.cc'); -// Run tools/generate_code_cache.js -const child = spawnSync( - process.execPath, - ['--expose-internals', generator, dest] -); +// Run mkcodecache +const child = spawnSync(generator, [dest]); assert.ifError(child.error); if (child.status !== 0) { console.log(child.stderr.toString()); diff --git a/tools/code_cache/cache_builder.cc b/tools/code_cache/cache_builder.cc new file mode 100644 index 0000000000..9ce4efa3a1 --- /dev/null +++ b/tools/code_cache/cache_builder.cc @@ -0,0 +1,165 @@ +#include "cache_builder.h" +#include <iostream> +#include <map> +#include <sstream> +#include <vector> +#include <cstdlib> +#include "util.h" + +#include "node_native_module.h" + +namespace node { +namespace native_module { + +using v8::Context; +using v8::Function; +using v8::Isolate; +using v8::Local; +using v8::MaybeLocal; +using v8::ScriptCompiler; + +static std::string GetDefName(const std::string& id) { + char buf[64] = {0}; + size_t size = id.size(); + CHECK_LT(size, sizeof(buf)); + for (size_t i = 0; i < size; ++i) { + char ch = id[i]; + buf[i] = (ch == '-' || ch == '/') ? '_' : ch; + } + return buf; +} + +static std::string FormatSize(size_t size) { + char buf[64] = {0}; + if (size < 1024) { + snprintf(buf, sizeof(buf), "%.2fB", static_cast<double>(size)); + } else if (size < 1024 * 1024) { + snprintf(buf, sizeof(buf), "%.2fKB", static_cast<double>(size / 1024)); + } else { + snprintf( + buf, sizeof(buf), "%.2fMB", static_cast<double>(size / 1024 / 1024)); + } + return buf; +} + +static std::string GetDefinition(const std::string& id, + size_t size, + const uint8_t* data) { + std::stringstream ss; + ss << "static const uint8_t " << GetDefName(id) << "[] = {\n"; + for (size_t i = 0; i < size; ++i) { + uint8_t ch = data[i]; + ss << std::to_string(ch) << (i == size - 1 ? '\n' : ','); + } + ss << "};"; + return ss.str(); +} + +static std::string GetInitializer(const std::string& id) { + std::string def_name = GetDefName(id); + char buf[256] = {0}; + snprintf(buf, + sizeof(buf), + "code_cache->emplace(\n" + " \"%s\",\n" + " std::make_unique<v8::ScriptCompiler::CachedData>" + "(%s, static_cast<int>(arraysize(%s)), policy)\n" + ");", + id.c_str(), + def_name.c_str(), + def_name.c_str()); + return buf; +} + +static std::string GenerateCodeCache( + std::map<std::string, ScriptCompiler::CachedData*> data, + std::vector<std::string> ids, + bool log_progress) { + std::stringstream ss; + ss << R"(#include <cinttypes> +#include "node_native_module_env.h" + +// This file is generated by tools/mkcodecache +// and is used when configure is run with \`--code-cache-path\` + +namespace node { +namespace native_module { +)"; + + size_t total = 0; + for (const auto& x : data) { + const std::string& id = x.first; + ScriptCompiler::CachedData* cached_data = x.second; + total += cached_data->length; + std::string def = GetDefinition(id, cached_data->length, cached_data->data); + ss << def << "\n\n"; + if (log_progress) { + std::cout << "Generated cache for " << id + << ", size = " << FormatSize(cached_data->length) + << ", total = " << FormatSize(total) << "\n"; + } + } + + ss << R"(void NativeModuleEnv::InitializeCodeCache() { + NativeModuleCacheMap* code_cache = + NativeModuleLoader::GetInstance()->code_cache(); + if (!code_cache->empty()) { + return; + } + auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned; +)"; + + for (const auto& x : data) { + const std::string& id = x.first; + ss << GetInitializer(id) << "\n\n"; + } + + ss << R"(} + +} // namespace native_module +} // namespace node +)"; + return ss.str(); +} + +std::string CodeCacheBuilder::Generate(Local<Context> context) { + NativeModuleLoader* loader = NativeModuleLoader::GetInstance(); + std::vector<std::string> ids = loader->GetModuleIds(); + + std::vector<std::string> modules; + modules.reserve(ids.size()); + + std::map<std::string, ScriptCompiler::CachedData*> data; + + NativeModuleLoader::Result result; + for (const auto& id : ids) { + // TODO(joyeecheung): we can only compile the modules that can be + // required here because the parameters for other types of builtins + // are still very flexible. We should look into auto-generating + // the paramters from the source somehow. + if (loader->CanBeRequired(id.c_str())) { + modules.push_back(id); + USE(loader->CompileAsModule(context, id.c_str(), &result)); + ScriptCompiler::CachedData* cached_data = + loader->GetCodeCache(id.c_str()); + if (cached_data == nullptr) { + // TODO(joyeecheung): display syntax errors + std::cerr << "Failed to complile " << id << "\n"; + } else { + data.emplace(id, cached_data); + } + } + } + + char env_buf[32]; + size_t env_size = sizeof(env_buf); + int ret = uv_os_getenv("NODE_DEBUG", env_buf, &env_size); + bool log_progress = false; + if (ret == 0 && strcmp(env_buf, "mkcodecache") == 0) { + log_progress = true; + } + return GenerateCodeCache(data, modules, log_progress); +} + +} // namespace native_module +} // namespace node diff --git a/tools/code_cache/cache_builder.h b/tools/code_cache/cache_builder.h new file mode 100644 index 0000000000..d5a6cd4241 --- /dev/null +++ b/tools/code_cache/cache_builder.h @@ -0,0 +1,16 @@ +#ifndef TOOLS_CODE_CACHE_CACHE_BUILDER_H_ +#define TOOLS_CODE_CACHE_CACHE_BUILDER_H_ + +#include <string> +#include "v8.h" + +namespace node { +namespace native_module { +class CodeCacheBuilder { + public: + static std::string Generate(v8::Local<v8::Context> context); +}; +} // namespace native_module +} // namespace node + +#endif // TOOLS_CODE_CACHE_CACHE_BUILDER_H_ diff --git a/tools/code_cache/mkcodecache.cc b/tools/code_cache/mkcodecache.cc new file mode 100644 index 0000000000..24f7e05e1f --- /dev/null +++ b/tools/code_cache/mkcodecache.cc @@ -0,0 +1,62 @@ +#include <cstdio> +#include <fstream> +#include <iostream> +#include <sstream> +#include <string> +#include <vector> + +#include "cache_builder.h" +#include "libplatform/libplatform.h" +#include "v8.h" + +using node::native_module::CodeCacheBuilder; +using v8::ArrayBuffer; +using v8::Context; +using v8::HandleScope; +using v8::Isolate; +using v8::Local; + +#ifdef _WIN32 +#include <VersionHelpers.h> +#include <WinError.h> +#include <windows.h> + +int wmain(int argc, wchar_t* argv[]) { +#else // UNIX +int main(int argc, char* argv[]) { +#endif // _WIN32 + + if (argc < 2) { + std::cerr << "Usage: " << argv[0] << " <path/to/output.cc>\n"; + return 1; + } + + std::ofstream out; + out.open(argv[1], std::ios::out | std::ios::binary); + if (!out.is_open()) { + std::cerr << "Cannot open " << argv[1] << "\n"; + return 1; + } + + std::unique_ptr<v8::Platform> platform = v8::platform::NewDefaultPlatform(); + v8::V8::InitializePlatform(platform.get()); + v8::V8::Initialize(); + + // Create a new Isolate and make it the current one. + Isolate::CreateParams create_params; + create_params.array_buffer_allocator = + ArrayBuffer::Allocator::NewDefaultAllocator(); + Isolate* isolate = Isolate::New(create_params); + { + Isolate::Scope isolate_scope(isolate); + v8::HandleScope handle_scope(isolate); + v8::Local<v8::Context> context = v8::Context::New(isolate); + v8::Context::Scope context_scope(context); + + std::string cache = CodeCacheBuilder::Generate(context); + out << cache; + out.close(); + } + + return 0; +} diff --git a/tools/generate_code_cache.js b/tools/generate_code_cache.js deleted file mode 100644 index 8afbc14a9b..0000000000 --- a/tools/generate_code_cache.js +++ /dev/null @@ -1,135 +0,0 @@ -'use strict'; - -// Flags: --expose-internals - -// This file generates the code cache for builtin modules and -// writes them into static char arrays of a C++ file that can be -// compiled into the binary using the `--code-cache-path` option -// of `configure`. - -const { internalBinding } = require('internal/test/binding'); -const { - moduleCategories: { canBeRequired }, - getCodeCache, - compileFunction, -} = internalBinding('native_module'); - -const { - types: { - isUint8Array - } -} = require('util'); - -const fs = require('fs'); - -const resultPath = process.argv[2]; -if (!resultPath) { - console.error(`Usage: ${process.argv[0]} ${process.argv[1]}` + - 'path/to/node_code_cache.cc'); - process.exit(1); -} - -/** - * Format a number of a size in bytes into human-readable strings - * @param {number} num - * @return {string} - */ -function formatSize(num) { - if (num < 1024) { - return `${(num).toFixed(2)}B`; - } else if (num < 1024 ** 2) { - return `${(num / 1024).toFixed(2)}KB`; - } else if (num < 1024 ** 3) { - return `${(num / (1024 ** 2)).toFixed(2)}MB`; - } else { - return `${(num / (1024 ** 3)).toFixed(2)}GB`; - } -} - -/** - * Generates the source code of definitions of the char arrays - * that contains the code cache and the source code of the - * initializers of the code cache. - * - * @param {string} key ID of the builtin module - * @param {Uint8Array} cache Code cache of the builtin module - * @return { definition: string, initializer: string } - */ -function getInitalizer(key, cache) { - const defName = `${key.replace(/\//g, '_').replace(/-/g, '_')}_raw`; - const definition = `static const uint8_t ${defName}[] = {\n` + - `${cache.join(',')}\n};`; - const dataDef = 'std::make_unique<v8::ScriptCompiler::CachedData>(' + - `${defName}, static_cast<int>(arraysize(${defName})), ` + - 'policy)'; - const initializer = - 'code_cache->emplace(\n' + - ` "${key}",\n` + - ` ${dataDef}\n` + - ');'; - return { - definition, initializer - }; -} - -const cacheDefinitions = []; -const cacheInitializers = []; -let totalCacheSize = 0; - -function lexical(a, b) { - if (a < b) { - return -1; - } - if (a > b) { - return 1; - } - return 0; -} - -// TODO(joyeecheung): support non-modules that require different -// parameters in the wrapper. -for (const key of [...canBeRequired].sort(lexical)) { - compileFunction(key); // compile it - const cachedData = getCodeCache(key); - if (!isUint8Array(cachedData)) { - console.error(`Failed to generate code cache for '${key}'`); - process.exit(1); - } - - const size = cachedData.byteLength; - totalCacheSize += size; - const { - definition, initializer, - } = getInitalizer(key, cachedData); - cacheDefinitions.push(definition); - cacheInitializers.push(initializer); - console.log(`Generated cache for '${key}', size = ${formatSize(size)}` + - `, total = ${formatSize(totalCacheSize)}`); -} - -const result = `#include "node_native_module_env.h" - -// This file is generated by tools/generate_code_cache.js -// and is used when configure is run with \`--code-cache-path\` - -namespace node { -namespace native_module { -${cacheDefinitions.join('\n\n')} - -void NativeModuleEnv::InitializeCodeCache() { - NativeModuleCacheMap* code_cache = - NativeModuleLoader::GetInstance()->code_cache(); - if (!code_cache->empty()) { - return; - } - - auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned; - ${cacheInitializers.join('\n ')} -} - -} // namespace native_module -} // namespace node -`; - -fs.writeFileSync(resultPath, result); -console.log(`Generated code cache C++ file to ${resultPath}`); |