tools: implement mkcodecache as an executable

This patch implement a mkcodecache executable on top of the
`NativeModuleLoader` singleton.
This makes it possible to build a Node.js binary with embedded
code cache without building itself using the code cache stub -
the cache is now initialized by `NativeModuleEnv` instead which
can be refactored out of the mkcodecache dependencies.

PR-URL: https://github.com/nodejs/node/pull/27161
Reviewed-By: Joyee Cheung <joyeec9h3@gmail.com>

4 files changed, 243 insertions, 135 deletions

diff --git a/tools/code_cache/cache_builder.cc b/tools/code_cache/cache_builder.cc
new file mode 100644
index 0000000000..9ce4efa3a1
--- /dev/null
+++ b/tools/code_cache/cache_builder.cc
@@ -0,0 +1,165 @@
+#include "cache_builder.h"
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <vector>
+#include <cstdlib>
+#include "util.h"
+
+#include "node_native_module.h"
+
+namespace node {
+namespace native_module {
+
+using v8::Context;
+using v8::Function;
+using v8::Isolate;
+using v8::Local;
+using v8::MaybeLocal;
+using v8::ScriptCompiler;
+
+static std::string GetDefName(const std::string& id) {
+  char buf[64] = {0};
+  size_t size = id.size();
+  CHECK_LT(size, sizeof(buf));
+  for (size_t i = 0; i < size; ++i) {
+    char ch = id[i];
+    buf[i] = (ch == '-' || ch == '/') ? '_' : ch;
+  }
+  return buf;
+}
+
+static std::string FormatSize(size_t size) {
+  char buf[64] = {0};
+  if (size < 1024) {
+    snprintf(buf, sizeof(buf), "%.2fB", static_cast<double>(size));
+  } else if (size < 1024 * 1024) {
+    snprintf(buf, sizeof(buf), "%.2fKB", static_cast<double>(size / 1024));
+  } else {
+    snprintf(
+        buf, sizeof(buf), "%.2fMB", static_cast<double>(size / 1024 / 1024));
+  }
+  return buf;
+}
+
+static std::string GetDefinition(const std::string& id,
+                                 size_t size,
+                                 const uint8_t* data) {
+  std::stringstream ss;
+  ss << "static const uint8_t " << GetDefName(id) << "[] = {\n";
+  for (size_t i = 0; i < size; ++i) {
+    uint8_t ch = data[i];
+    ss << std::to_string(ch) << (i == size - 1 ? '\n' : ',');
+  }
+  ss << "};";
+  return ss.str();
+}
+
+static std::string GetInitializer(const std::string& id) {
+  std::string def_name = GetDefName(id);
+  char buf[256] = {0};
+  snprintf(buf,
+           sizeof(buf),
+           "code_cache->emplace(\n"
+           "  \"%s\",\n"
+           "  std::make_unique<v8::ScriptCompiler::CachedData>"
+           "(%s, static_cast<int>(arraysize(%s)), policy)\n"
+           ");",
+           id.c_str(),
+           def_name.c_str(),
+           def_name.c_str());
+  return buf;
+}
+
+static std::string GenerateCodeCache(
+    std::map<std::string, ScriptCompiler::CachedData*> data,
+    std::vector<std::string> ids,
+    bool log_progress) {
+  std::stringstream ss;
+  ss << R"(#include <cinttypes>
+#include "node_native_module_env.h"
+
+// This file is generated by tools/mkcodecache
+// and is used when configure is run with \`--code-cache-path\`
+
+namespace node {
+namespace native_module {
+)";
+
+  size_t total = 0;
+  for (const auto& x : data) {
+    const std::string& id = x.first;
+    ScriptCompiler::CachedData* cached_data = x.second;
+    total += cached_data->length;
+    std::string def = GetDefinition(id, cached_data->length, cached_data->data);
+    ss << def << "\n\n";
+    if (log_progress) {
+      std::cout << "Generated cache for " << id
+                << ", size = " << FormatSize(cached_data->length)
+                << ", total = " << FormatSize(total) << "\n";
+    }
+  }
+
+  ss << R"(void NativeModuleEnv::InitializeCodeCache() {
+  NativeModuleCacheMap* code_cache =
+      NativeModuleLoader::GetInstance()->code_cache();
+  if (!code_cache->empty()) {
+    return;
+  }
+  auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned;
+)";
+
+  for (const auto& x : data) {
+    const std::string& id = x.first;
+    ss << GetInitializer(id) << "\n\n";
+  }
+
+  ss << R"(}
+
+}  // namespace native_module
+}  // namespace node
+)";
+  return ss.str();
+}
+
+std::string CodeCacheBuilder::Generate(Local<Context> context) {
+  NativeModuleLoader* loader = NativeModuleLoader::GetInstance();
+  std::vector<std::string> ids = loader->GetModuleIds();
+
+  std::vector<std::string> modules;
+  modules.reserve(ids.size());
+
+  std::map<std::string, ScriptCompiler::CachedData*> data;
+
+  NativeModuleLoader::Result result;
+  for (const auto& id : ids) {
+    // TODO(joyeecheung): we can only compile the modules that can be
+    // required here because the parameters for other types of builtins
+    // are still very flexible. We should look into auto-generating
+    // the paramters from the source somehow.
+    if (loader->CanBeRequired(id.c_str())) {
+      modules.push_back(id);
+      USE(loader->CompileAsModule(context, id.c_str(), &result));
+      ScriptCompiler::CachedData* cached_data =
+          loader->GetCodeCache(id.c_str());
+      if (cached_data == nullptr) {
+        // TODO(joyeecheung): display syntax errors
+        std::cerr << "Failed to complile " << id << "\n";
+      } else {
+        data.emplace(id, cached_data);
+      }
+    }
+  }
+
+  char env_buf[32];
+  size_t env_size = sizeof(env_buf);
+  int ret = uv_os_getenv("NODE_DEBUG", env_buf, &env_size);
+  bool log_progress = false;
+  if (ret == 0 && strcmp(env_buf, "mkcodecache") == 0) {
+    log_progress = true;
+  }
+  return GenerateCodeCache(data, modules, log_progress);
+}
+
+}  // namespace native_module
+}  // namespace node
diff --git a/tools/code_cache/cache_builder.h b/tools/code_cache/cache_builder.h
new file mode 100644
index 0000000000..d5a6cd4241
--- /dev/null
+++ b/tools/code_cache/cache_builder.h
@@ -0,0 +1,16 @@
+#ifndef TOOLS_CODE_CACHE_CACHE_BUILDER_H_
+#define TOOLS_CODE_CACHE_CACHE_BUILDER_H_
+
+#include <string>
+#include "v8.h"
+
+namespace node {
+namespace native_module {
+class CodeCacheBuilder {
+ public:
+  static std::string Generate(v8::Local<v8::Context> context);
+};
+}  // namespace native_module
+}  // namespace node
+
+#endif  // TOOLS_CODE_CACHE_CACHE_BUILDER_H_
diff --git a/tools/code_cache/mkcodecache.cc b/tools/code_cache/mkcodecache.cc
new file mode 100644
index 0000000000..24f7e05e1f
--- /dev/null
+++ b/tools/code_cache/mkcodecache.cc
@@ -0,0 +1,62 @@
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include "cache_builder.h"
+#include "libplatform/libplatform.h"
+#include "v8.h"
+
+using node::native_module::CodeCacheBuilder;
+using v8::ArrayBuffer;
+using v8::Context;
+using v8::HandleScope;
+using v8::Isolate;
+using v8::Local;
+
+#ifdef _WIN32
+#include <VersionHelpers.h>
+#include <WinError.h>
+#include <windows.h>
+
+int wmain(int argc, wchar_t* argv[]) {
+#else   // UNIX
+int main(int argc, char* argv[]) {
+#endif  // _WIN32
+
+  if (argc < 2) {
+    std::cerr << "Usage: " << argv[0] << " <path/to/output.cc>\n";
+    return 1;
+  }
+
+  std::ofstream out;
+  out.open(argv[1], std::ios::out | std::ios::binary);
+  if (!out.is_open()) {
+    std::cerr << "Cannot open " << argv[1] << "\n";
+    return 1;
+  }
+
+  std::unique_ptr<v8::Platform> platform = v8::platform::NewDefaultPlatform();
+  v8::V8::InitializePlatform(platform.get());
+  v8::V8::Initialize();
+
+  // Create a new Isolate and make it the current one.
+  Isolate::CreateParams create_params;
+  create_params.array_buffer_allocator =
+      ArrayBuffer::Allocator::NewDefaultAllocator();
+  Isolate* isolate = Isolate::New(create_params);
+  {
+    Isolate::Scope isolate_scope(isolate);
+    v8::HandleScope handle_scope(isolate);
+    v8::Local<v8::Context> context = v8::Context::New(isolate);
+    v8::Context::Scope context_scope(context);
+
+    std::string cache = CodeCacheBuilder::Generate(context);
+    out << cache;
+    out.close();
+  }
+
+  return 0;
+}
diff --git a/tools/generate_code_cache.js b/tools/generate_code_cache.js
deleted file mode 100644
index 8afbc14a9b..0000000000
--- a/tools/generate_code_cache.js
+++ /dev/null
@@ -1,135 +0,0 @@
-'use strict';
-
-// Flags: --expose-internals
-
-// This file generates the code cache for builtin modules and
-// writes them into static char arrays of a C++ file that can be
-// compiled into the binary using the `--code-cache-path` option
-// of `configure`.
-
-const { internalBinding } = require('internal/test/binding');
-const {
-  moduleCategories: { canBeRequired },
-  getCodeCache,
-  compileFunction,
-} = internalBinding('native_module');
-
-const {
-  types: {
-    isUint8Array
-  }
-} = require('util');
-
-const fs = require('fs');
-
-const resultPath = process.argv[2];
-if (!resultPath) {
-  console.error(`Usage: ${process.argv[0]} ${process.argv[1]}` +
-                'path/to/node_code_cache.cc');
-  process.exit(1);
-}
-
-/**
- * Format a number of a size in bytes into human-readable strings
- * @param {number} num
- * @return {string}
- */
-function formatSize(num) {
-  if (num < 1024) {
-    return `${(num).toFixed(2)}B`;
-  } else if (num < 1024 ** 2) {
-    return `${(num / 1024).toFixed(2)}KB`;
-  } else if (num < 1024 ** 3) {
-    return `${(num / (1024 ** 2)).toFixed(2)}MB`;
-  } else {
-    return `${(num / (1024 ** 3)).toFixed(2)}GB`;
-  }
-}
-
-/**
- * Generates the source code of definitions of the char arrays
- * that contains the code cache and the source code of the
- * initializers of the code cache.
- *
- * @param {string} key ID of the builtin module
- * @param {Uint8Array} cache Code cache of the builtin module
- * @return { definition: string, initializer: string }
- */
-function getInitalizer(key, cache) {
-  const defName = `${key.replace(/\//g, '_').replace(/-/g, '_')}_raw`;
-  const definition = `static const uint8_t ${defName}[] = {\n` +
-                     `${cache.join(',')}\n};`;
-  const dataDef = 'std::make_unique<v8::ScriptCompiler::CachedData>(' +
-                  `${defName}, static_cast<int>(arraysize(${defName})), ` +
-                  'policy)';
-  const initializer =
-    'code_cache->emplace(\n' +
-    `  "${key}",\n` +
-    `  ${dataDef}\n` +
-    ');';
-  return {
-    definition, initializer
-  };
-}
-
-const cacheDefinitions = [];
-const cacheInitializers = [];
-let totalCacheSize = 0;
-
-function lexical(a, b) {
-  if (a < b) {
-    return -1;
-  }
-  if (a > b) {
-    return 1;
-  }
-  return 0;
-}
-
-// TODO(joyeecheung): support non-modules that require different
-// parameters in the wrapper.
-for (const key of [...canBeRequired].sort(lexical)) {
-  compileFunction(key);  // compile it
-  const cachedData = getCodeCache(key);
-  if (!isUint8Array(cachedData)) {
-    console.error(`Failed to generate code cache for '${key}'`);
-    process.exit(1);
-  }
-
-  const size = cachedData.byteLength;
-  totalCacheSize += size;
-  const {
-    definition, initializer,
-  } = getInitalizer(key, cachedData);
-  cacheDefinitions.push(definition);
-  cacheInitializers.push(initializer);
-  console.log(`Generated cache for '${key}', size = ${formatSize(size)}` +
-              `, total = ${formatSize(totalCacheSize)}`);
-}
-
-const result = `#include "node_native_module_env.h"
-
-// This file is generated by tools/generate_code_cache.js
-// and is used when configure is run with \`--code-cache-path\`
-
-namespace node {
-namespace native_module {
-${cacheDefinitions.join('\n\n')}
-
-void NativeModuleEnv::InitializeCodeCache() {
-  NativeModuleCacheMap* code_cache =
-      NativeModuleLoader::GetInstance()->code_cache();
-  if (!code_cache->empty()) {
-    return;
-  }
-
-  auto policy = v8::ScriptCompiler::CachedData::BufferPolicy::BufferNotOwned;
-  ${cacheInitializers.join('\n  ')}
-}
-
-}  // namespace native_module
-}  // namespace node
-`;
-
-fs.writeFileSync(resultPath, result);
-console.log(`Generated code cache C++ file to ${resultPath}`);