diff options
| -rw-r--r-- | Makefile | 16 | ||||
| -rwxr-xr-x | configure | 8 | ||||
| -rw-r--r-- | lib/internal/bootstrap/cache.js | 30 | ||||
| -rw-r--r-- | lib/internal/bootstrap/loaders.js | 29 | ||||
| -rw-r--r-- | node.gyp | 8 | ||||
| -rw-r--r-- | src/node.cc | 13 | ||||
| -rw-r--r-- | src/node_code_cache.h | 16 | ||||
| -rw-r--r-- | src/node_code_cache_stub.cc | 14 | ||||
| -rw-r--r-- | test/code-cache/code-cache.status | 21 | ||||
| -rw-r--r-- | test/code-cache/test-code-cache.js | 42 | ||||
| -rw-r--r-- | test/code-cache/testcfg.py | 6 | ||||
| -rw-r--r-- | tools/generate_code_cache.js | 124 | ||||
| -rwxr-xr-x | tools/test.py | 1 |
13 files changed, 325 insertions, 3 deletions
@@ -91,6 +91,22 @@ $(NODE_G_EXE): config.gypi out/Makefile $(MAKE) -C out BUILDTYPE=Debug V=$(V) if [ ! -r $@ -o ! -L $@ ]; then ln -fs out/Debug/$(NODE_EXE) $@; fi +CODE_CACHE_DIR ?= out/$(BUILDTYPE)/obj/gen +CODE_CACHE_FILE ?= $(CODE_CACHE_DIR)/node_code_cache.cc + +.PHONY: with-code-cache +with-code-cache: + $(PYTHON) ./configure + $(MAKE) + mkdir -p $(CODE_CACHE_DIR) + out/$(BUILDTYPE)/$(NODE_EXE) --expose-internals tools/generate_code_cache.js $(CODE_CACHE_FILE) + $(PYTHON) ./configure --code-cache-path $(CODE_CACHE_FILE) + $(MAKE) + +.PHONY: test-code-cache +test-code-cache: with-code-cache + $(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) code-cache + out/Makefile: common.gypi deps/uv/uv.gyp deps/http_parser/http_parser.gyp \ deps/zlib/zlib.gyp deps/v8/gypfiles/toolchain.gypi \ deps/v8/gypfiles/features.gypi deps/v8/gypfiles/v8.gyp node.gyp \ @@ -491,6 +491,12 @@ parser.add_option('--without-snapshot', dest='without_snapshot', help=optparse.SUPPRESS_HELP) +parser.add_option('--code-cache-path', + action='store', + dest='code_cache_path', + help='Use a file generated by tools/generate_code_cache.js to compile the' + ' code cache for builtin modules into the binary') + parser.add_option('--without-ssl', action='store_true', dest='without_ssl', @@ -983,6 +989,8 @@ def configure_node(o): o['variables']['debug_nghttp2'] = 'false' o['variables']['node_no_browser_globals'] = b(options.no_browser_globals) + if options.code_cache_path: + o['variables']['node_code_cache_path'] = options.code_cache_path o['variables']['node_shared'] = b(options.shared) node_module_version = getmoduleversion.get_version() diff --git a/lib/internal/bootstrap/cache.js b/lib/internal/bootstrap/cache.js new file mode 100644 index 0000000000..a3d22ba020 --- /dev/null +++ b/lib/internal/bootstrap/cache.js @@ -0,0 +1,30 @@ +'use strict'; + +// This is only exposed for internal build steps and testing purposes. +// We create new copies of the source and the code cache +// so the resources eventually used to compile builtin modules +// cannot be tampered with even with --expose-internals + +const { + NativeModule, internalBinding +} = require('internal/bootstrap/loaders'); + +module.exports = { + builtinSource: Object.assign({}, NativeModule._source), + codeCache: internalBinding('code_cache'), + compiledWithoutCache: NativeModule.compiledWithoutCache, + compiledWithCache: NativeModule.compiledWithCache, + nativeModuleWrap(script) { + return NativeModule.wrap(script); + }, + // Modules with source code compiled in js2c that + // cannot be compiled with the code cache + cannotUseCache: [ + 'config', + // TODO(joyeecheung): update the C++ side so that + // the code cache is also used when compiling these + // two files. + 'internal/bootstrap/loaders', + 'internal/bootstrap/node' + ] +}; diff --git a/lib/internal/bootstrap/loaders.js b/lib/internal/bootstrap/loaders.js index de911eb841..c141c9adcf 100644 --- a/lib/internal/bootstrap/loaders.js +++ b/lib/internal/bootstrap/loaders.js @@ -125,10 +125,15 @@ const config = getBinding('config'); + const codeCache = getInternalBinding('code_cache'); + const compiledWithoutCache = NativeModule.compiledWithoutCache = []; + const compiledWithCache = NativeModule.compiledWithCache = []; + // Think of this as module.exports in this file even though it is not // written in CommonJS style. const loaderExports = { internalBinding, NativeModule }; const loaderId = 'internal/bootstrap/loaders'; + NativeModule.require = function(id) { if (id === loaderId) { return loaderExports; @@ -229,7 +234,29 @@ this.loading = true; try { - const script = new ContextifyScript(source, this.filename); + // (code, filename, lineOffset, columnOffset + // cachedData, produceCachedData, parsingContext) + const script = new ContextifyScript( + source, this.filename, 0, 0, + codeCache[this.id], false, undefined + ); + + // One of these conditions may be false when any of the inputs + // of the `node_js2c` target in node.gyp is modified. + // FIXME(joyeecheung): + // 1. Figure out how to resolve the dependency issue. When the + // code cache was introduced we were at a point where refactoring + // node.gyp may not be worth the effort. + // 2. Calculate checksums in both js2c and generate_code_cache.js + // and compare them before compiling the native modules since + // V8 only checks the length of the source to decide whether to + // reject the cache. + if (!codeCache[this.id] || script.cachedDataRejected) { + compiledWithoutCache.push(this.id); + } else { + compiledWithCache.push(this.id); + } + // Arguments: timeout, displayErrors, breakOnSigint const fn = script.runInThisContext(-1, true, false); const requireFn = this.id.startsWith('internal/deps/') ? @@ -6,6 +6,7 @@ 'node_use_etw%': 'false', 'node_use_perfctr%': 'false', 'node_no_browser_globals%': 'false', + 'node_code_cache_path%': '', 'node_use_v8_platform%': 'true', 'node_use_bundled_v8%': 'true', 'node_shared%': 'false', @@ -24,6 +25,7 @@ 'node_lib_target_name%': 'node_lib', 'node_intermediate_lib_type%': 'static_library', 'library_files': [ + 'lib/internal/bootstrap/cache.js', 'lib/internal/bootstrap/loaders.js', 'lib/internal/bootstrap/node.js', 'lib/async_hooks.js', @@ -396,6 +398,7 @@ 'src/module_wrap.h', 'src/node.h', 'src/node_buffer.h', + 'src/node_code_cache.h', 'src/node_constants.h', 'src/node_contextify.h', 'src/node_debug_options.h', @@ -459,6 +462,11 @@ 'NODE_OPENSSL_SYSTEM_CERT_PATH="<(openssl_system_ca_path)"', ], 'conditions': [ + [ 'node_code_cache_path!=""', { + 'sources': [ '<(node_code_cache_path)' ] + }, { + 'sources': [ 'src/node_code_cache_stub.cc' ] + }], [ 'node_shared=="true" and node_module_version!="" and OS!="win"', { 'product_extension': '<(shlib_suffix)', 'xcode_settings': { diff --git a/src/node.cc b/src/node.cc index 513e95f2ed..acc5ab3a1e 100644 --- a/src/node.cc +++ b/src/node.cc @@ -22,6 +22,7 @@ #include "node_buffer.h" #include "node_constants.h" #include "node_javascript.h" +#include "node_code_cache.h" #include "node_platform.h" #include "node_version.h" #include "node_internals.h" @@ -1596,10 +1597,18 @@ static void GetInternalBinding(const FunctionCallbackInfo<Value>& args) { Local<String> module = args[0].As<String>(); node::Utf8Value module_v(env->isolate(), module); + Local<Object> exports; node_module* mod = get_internal_module(*module_v); - if (mod == nullptr) return ThrowIfNoSuchModule(env, *module_v); - Local<Object> exports = InitModule(env, mod, module); + if (mod != nullptr) { + exports = InitModule(env, mod, module); + } else if (!strcmp(*module_v, "code_cache")) { + // internalBinding('code_cache') + exports = Object::New(env->isolate()); + DefineCodeCache(env, exports); + } else { + return ThrowIfNoSuchModule(env, *module_v); + } args.GetReturnValue().Set(exports); } diff --git a/src/node_code_cache.h b/src/node_code_cache.h new file mode 100644 index 0000000000..d4775a2b65 --- /dev/null +++ b/src/node_code_cache.h @@ -0,0 +1,16 @@ +#ifndef SRC_NODE_CODE_CACHE_H_ +#define SRC_NODE_CODE_CACHE_H_ + +#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#include "node_internals.h" + +namespace node { + +void DefineCodeCache(Environment* env, v8::Local<v8::Object> target); + +} // namespace node + +#endif // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS + +#endif // SRC_NODE_CODE_CACHE_H_ diff --git a/src/node_code_cache_stub.cc b/src/node_code_cache_stub.cc new file mode 100644 index 0000000000..3e9e5960c1 --- /dev/null +++ b/src/node_code_cache_stub.cc @@ -0,0 +1,14 @@ + +#include "node_code_cache.h" + +// This is supposed to be generated by tools/generate_code_cache.js +// The stub here is used when configure is run without `--code-cache-path` + +namespace node { +void DefineCodeCache(Environment* env, v8::Local<v8::Object> target) { + // When we do not produce code cache for builtin modules, + // `internalBinding('code_cache')` returns an empty object + // (here as `target`) so this is a noop. +} + +} // namespace node diff --git a/test/code-cache/code-cache.status b/test/code-cache/code-cache.status new file mode 100644 index 0000000000..26eb66ea2d --- /dev/null +++ b/test/code-cache/code-cache.status @@ -0,0 +1,21 @@ +prefix code-cache + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$arch==arm || $arch==arm64] + +[$system==solaris] # Also applies to SmartOS + +[$system==freebsd] + +[$system==aix] diff --git a/test/code-cache/test-code-cache.js b/test/code-cache/test-code-cache.js new file mode 100644 index 0000000000..a437834301 --- /dev/null +++ b/test/code-cache/test-code-cache.js @@ -0,0 +1,42 @@ +'use strict'; + +// Flags: --expose-internals +// This test verifies that the binary is compiled with code cache and the +// cache is used when built in modules are compiled. + +require('../common'); +const assert = require('assert'); +const { + types: { + isUint8Array + } +} = require('util'); +const { + builtinSource, + codeCache, + cannotUseCache, + compiledWithCache, + compiledWithoutCache +} = require('internal/bootstrap/cache'); + +assert.strictEqual( + typeof process.config.variables.node_code_cache_path, + 'string' +); + +assert.deepStrictEqual(compiledWithoutCache, []); + +const loadedModules = process.moduleLoadList + .filter((m) => m.startsWith('NativeModule')) + .map((m) => m.replace('NativeModule ', '')); + +for (const key of loadedModules) { + assert(compiledWithCache.includes(key), + `"${key}" should've been compiled with code cache`); +} + +for (const key of Object.keys(builtinSource)) { + if (cannotUseCache.includes(key)) continue; + assert(isUint8Array(codeCache[key]) && codeCache[key].length > 0, + `Code cache for "${key}" should've been generated`); +} diff --git a/test/code-cache/testcfg.py b/test/code-cache/testcfg.py new file mode 100644 index 0000000000..0e0035dfed --- /dev/null +++ b/test/code-cache/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.ParallelTestConfiguration(context, root, 'code-cache') diff --git a/tools/generate_code_cache.js b/tools/generate_code_cache.js new file mode 100644 index 0000000000..8aab6bc286 --- /dev/null +++ b/tools/generate_code_cache.js @@ -0,0 +1,124 @@ +'use strict'; + +// Flags: --expose-internals + +// This file generates the code cache for builtin modules and +// writes them into static char arrays of a C++ file that can be +// compiled into the binary using the `--code-cache-path` option +// of `configure`. + +const { + nativeModuleWrap, + builtinSource, + cannotUseCache +} = require('internal/bootstrap/cache'); + +const vm = require('vm'); +const fs = require('fs'); + +const resultPath = process.argv[2]; +if (!resultPath) { + console.error(`Usage: ${process.argv[0]} ${process.argv[1]}` + + 'path/to/node_code_cache.cc'); + process.exit(1); +} + +/** + * Format a number of a size in bytes into human-readable strings + * @param {number} num + * @return {string} + */ +function formatSize(num) { + if (num < 1024) { + return `${(num).toFixed(2)}B`; + } else if (num < 1024 ** 2) { + return `${(num / 1024).toFixed(2)}KB`; + } else if (num < 1024 ** 3) { + return `${(num / (1024 ** 2)).toFixed(2)}MB`; + } else { + return `${(num / (1024 ** 3)).toFixed(2)}GB`; + } +} + +/** + * Generates the source code of definitions of the char arrays + * that contains the code cache and the source code of the + * initializers of the code cache. + * + * @param {string} key ID of the builtin module + * @param {Buffer} cache Code cache of the builtin module + * @return { definition: string, initializer: string } + */ +function getInitalizer(key, cache) { + const defName = key.replace(/\//g, '_').replace(/-/g, '_'); + const definition = `static uint8_t ${defName}_raw[] = {\n` + + `${cache.join(',')}\n};`; + const initializer = ` + v8::Local<v8::ArrayBuffer> ${defName}_ab = + v8::ArrayBuffer::New(isolate, ${defName}_raw, ${cache.length}); + v8::Local<v8::Uint8Array> ${defName}_array = + v8::Uint8Array::New(${defName}_ab, 0, ${cache.length}); + target->Set(context, + FIXED_ONE_BYTE_STRING(isolate, "${key}"), + ${defName}_array).FromJust(); + `; + return { + definition, initializer + }; +} + +const cacheDefinitions = []; +const cacheInitializers = []; +let totalCacheSize = 0; + + +for (const key of Object.keys(builtinSource)) { + if (cannotUseCache.includes(key)) continue; + const code = nativeModuleWrap(builtinSource[key]); + + // Note that this must corresponds to the code in + // NativeModule.prototype.compile + const script = new vm.Script(code, { + filename: `${key}.js`, + produceCachedData: true + }); + + if (!script.cachedData) { + console.error(`Failed to generate code cache for '${key}'`); + process.exit(1); + } + + const length = script.cachedData.length; + totalCacheSize += length; + const { definition, initializer } = getInitalizer(key, script.cachedData); + cacheDefinitions.push(definition); + cacheInitializers.push(initializer); + console.log(`Generated cache for '${key}', size = ${formatSize(length)}` + + `, total = ${formatSize(totalCacheSize)}`); +} + +const result = `#include "node.h" +#include "node_code_cache.h" +#include "v8.h" +#include "env.h" +#include "env-inl.h" + +// This file is generated by tools/generate_code_cache.js +// and is used when configure is run with \`--code-cache-path\` + +namespace node { + +${cacheDefinitions.join('\n\n')} + +// The target here will be returned as \`internalBinding('code_cache')\` +void DefineCodeCache(Environment* env, v8::Local<v8::Object> target) { + v8::Isolate* isolate = env->isolate(); + v8::Local<v8::Context> context = env->context(); + ${cacheInitializers.join('\n')} +} + +} // namespace node +`; + +fs.writeFileSync(resultPath, result); +console.log(`Generated code cache C++ file to ${resultPath}`); diff --git a/tools/test.py b/tools/test.py index 5bda606cc8..c963196c69 100755 --- a/tools/test.py +++ b/tools/test.py @@ -1549,6 +1549,7 @@ def PrintCrashed(code): IGNORED_SUITES = [ 'addons', 'addons-napi', + 'code-cache', 'doctool', 'gc', 'internet', |