module: resolve and instantiate loader pipeline hooks

This enables a --loader flag for Node, which can provide custom
"resolve" and "dynamicInstantiate" methods for custom ES module
loading.

In the process, module providers have been converted from classes
into functions and the module APIs have been made to pass URL strings
over objects.

PR-URL: https://github.com/nodejs/node/pull/15445
Reviewed-By: Bradley Farias <bradley.meck@gmail.com>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Jeremiah Senkpiel <fishrock123@rocketmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>
Reviewed-By: Timothy Gu <timothygu99@gmail.com>

7 files changed, 236 insertions, 194 deletions

diff --git a/lib/internal/errors.js b/lib/internal/errors.js
index f49bb2e542..9a5551617d 100755
--- a/lib/internal/errors.js
+++ b/lib/internal/errors.js
@@ -328,6 +328,8 @@ E('ERR_TRANSFORM_WITH_LENGTH_0',
 E('ERR_UNESCAPED_CHARACTERS',
   (name) => `${name} contains unescaped characters`);
 E('ERR_UNKNOWN_ENCODING', 'Unknown encoding: %s');
+E('ERR_UNKNOWN_FILE_EXTENSION', 'Unknown file extension: %s');
+E('ERR_UNKNOWN_MODULE_FORMAT', 'Unknown module format: %s');
 E('ERR_UNKNOWN_SIGNAL', 'Unknown signal: %s');
 E('ERR_UNKNOWN_STDIN_TYPE', 'Unknown stdin file type');
 E('ERR_UNKNOWN_STREAM_TYPE', 'Unknown stream file type');
diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js
index a409d397f8..57c70188d6 100644
--- a/lib/internal/loader/Loader.js
+++ b/lib/internal/loader/Loader.js
@@ -1,20 +1,21 @@
 'use strict';
 
-const { URL } = require('url');
 const { getURLFromFilePath } = require('internal/url');
 
 const {
-  getNamespaceOfModuleWrap
+  getNamespaceOfModuleWrap,
+  createDynamicModule
 } = require('internal/loader/ModuleWrap');
 
 const ModuleMap = require('internal/loader/ModuleMap');
 const ModuleJob = require('internal/loader/ModuleJob');
-const resolveRequestUrl = require('internal/loader/resolveRequestUrl');
+const ModuleRequest = require('internal/loader/ModuleRequest');
 const errors = require('internal/errors');
+const debug = require('util').debuglog('esm');
 
 function getBase() {
   try {
-    return getURLFromFilePath(`${process.cwd()}/`);
+    return getURLFromFilePath(`${process.cwd()}/`).href;
   } catch (e) {
     e.stack;
     // If the current working directory no longer exists.
@@ -28,45 +29,75 @@ function getBase() {
 class Loader {
   constructor(base = getBase()) {
     this.moduleMap = new ModuleMap();
-    if (typeof base !== 'undefined' && base instanceof URL !== true) {
-      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL');
+    if (typeof base !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'string');
     }
     this.base = base;
+    this.resolver = ModuleRequest.resolve.bind(null);
+    this.dynamicInstantiate = undefined;
   }
 
-  async resolve(specifier) {
-    const request = resolveRequestUrl(this.base, specifier);
-    if (request.url.protocol !== 'file:') {
-      throw new errors.Error('ERR_INVALID_PROTOCOL',
-                             request.url.protocol, 'file:');
-    }
-    return request.url;
+  hook({ resolve = ModuleRequest.resolve, dynamicInstantiate }) {
+    this.resolver = resolve.bind(null);
+    this.dynamicInstantiate = dynamicInstantiate;
   }
 
-  async getModuleJob(dependentJob, specifier) {
-    if (!this.moduleMap.has(dependentJob.url)) {
-      throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url);
+  async resolve(specifier, parentURL = this.base) {
+    if (typeof parentURL !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE',
+                                 'parentURL', 'string');
+    }
+    const { url, format } = await this.resolver(specifier, parentURL,
+                                                ModuleRequest.resolve);
+
+    if (typeof format !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'format',
+                                 ['esm', 'cjs', 'builtin', 'addon', 'json']);
+    }
+    if (typeof url !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string');
+    }
+
+    if (format === 'builtin') {
+      return { url: `node:${url}`, format };
     }
-    const request = await resolveRequestUrl(dependentJob.url, specifier);
-    const url = `${request.url}`;
-    if (this.moduleMap.has(url)) {
-      return this.moduleMap.get(url);
+
+    if (format !== 'dynamic') {
+      if (!ModuleRequest.loaders.has(format)) {
+        throw new errors.Error('ERR_UNKNOWN_MODULE_FORMAT', format);
+      }
+      if (!url.startsWith('file:')) {
+        throw new errors.Error('ERR_INVALID_PROTOCOL', url, 'file:');
+      }
     }
-    const dependencyJob = new ModuleJob(this, request);
-    this.moduleMap.set(url, dependencyJob);
-    return dependencyJob;
+
+    return { url, format };
   }
 
-  async import(specifier) {
-    const request = await resolveRequestUrl(this.base, specifier);
-    const url = `${request.url}`;
-    let job;
-    if (this.moduleMap.has(url)) {
-      job = this.moduleMap.get(url);
-    } else {
-      job = new ModuleJob(this, request);
+  async getModuleJob(specifier, parentURL = this.base) {
+    const { url, format } = await this.resolve(specifier, parentURL);
+    let job = this.moduleMap.get(url);
+    if (job === undefined) {
+      let loaderInstance;
+      if (format === 'dynamic') {
+        loaderInstance = async (url) => {
+          const { exports, execute } = await this.dynamicInstantiate(url);
+          return createDynamicModule(exports, url, (reflect) => {
+            debug(`Loading custom loader ${url}`);
+            execute(reflect.exports);
+          });
+        };
+      } else {
+        loaderInstance = ModuleRequest.loaders.get(format);
+      }
+      job = new ModuleJob(this, url, loaderInstance);
       this.moduleMap.set(url, job);
     }
+    return job;
+  }
+
+  async import(specifier, parentURL = this.base) {
+    const job = await this.getModuleJob(specifier, parentURL);
     const module = await job.run();
     return getNamespaceOfModuleWrap(module);
   }
diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js
index db4cb6ae5c..04d6111b87 100644
--- a/lib/internal/loader/ModuleJob.js
+++ b/lib/internal/loader/ModuleJob.js
@@ -2,51 +2,40 @@
 
 const { SafeSet, SafePromise } = require('internal/safe_globals');
 const resolvedPromise = SafePromise.resolve();
-const resolvedArrayPromise = SafePromise.resolve([]);
-const { ModuleWrap } = require('internal/loader/ModuleWrap');
 
-const NOOP = () => { /* No-op */ };
 class ModuleJob {
   /**
    * @param {module: ModuleWrap?, compiled: Promise} moduleProvider
    */
-  constructor(loader, moduleProvider, url) {
-    this.url = `${moduleProvider.url}`;
-    this.moduleProvider = moduleProvider;
+  constructor(loader, url, moduleProvider) {
     this.loader = loader;
     this.error = null;
     this.hadError = false;
 
-    if (moduleProvider instanceof ModuleWrap !== true) {
-      // linked == promise for dependency jobs, with module populated,
-      // module wrapper linked
-      this.modulePromise = this.moduleProvider.createModule();
-      this.module = undefined;
-      const linked = async () => {
-        const dependencyJobs = [];
-        this.module = await this.modulePromise;
-        this.module.link(async (dependencySpecifier) => {
-          const dependencyJobPromise =
-            this.loader.getModuleJob(this, dependencySpecifier);
-          dependencyJobs.push(dependencyJobPromise);
-          const dependencyJob = await dependencyJobPromise;
-          return dependencyJob.modulePromise;
-        });
-        return SafePromise.all(dependencyJobs);
-      };
-      this.linked = linked();
+    // linked == promise for dependency jobs, with module populated,
+    // module wrapper linked
+    this.moduleProvider = moduleProvider;
+    this.modulePromise = this.moduleProvider(url);
+    this.module = undefined;
+    this.reflect = undefined;
+    const linked = async () => {
+      const dependencyJobs = [];
+      ({ module: this.module,
+         reflect: this.reflect } = await this.modulePromise);
+      this.module.link(async (dependencySpecifier) => {
+        const dependencyJobPromise =
+            this.loader.getModuleJob(dependencySpecifier, url);
+        dependencyJobs.push(dependencyJobPromise);
+        const dependencyJob = await dependencyJobPromise;
+        return (await dependencyJob.modulePromise).module;
+      });
+      return SafePromise.all(dependencyJobs);
+    };
+    this.linked = linked();
 
-      // instantiated == deep dependency jobs wrappers instantiated,
-      //module wrapper instantiated
-      this.instantiated = undefined;
-    } else {
-      const getModuleProvider = async () => moduleProvider;
-      this.modulePromise = getModuleProvider();
-      this.moduleProvider = { finish: NOOP };
-      this.module = moduleProvider;
-      this.linked = resolvedArrayPromise;
-      this.instantiated = this.modulePromise;
-    }
+    // instantiated == deep dependency jobs wrappers instantiated,
+    // module wrapper instantiated
+    this.instantiated = undefined;
   }
 
   instantiate() {
diff --git a/lib/internal/loader/ModuleRequest.js b/lib/internal/loader/ModuleRequest.js
new file mode 100644
index 0000000000..88e48ae9d3
--- /dev/null
+++ b/lib/internal/loader/ModuleRequest.js
@@ -0,0 +1,122 @@
+'use strict';
+
+const fs = require('fs');
+const internalCJSModule = require('internal/module');
+const internalURLModule = require('internal/url');
+const internalFS = require('internal/fs');
+const NativeModule = require('native_module');
+const { extname, _makeLong } = require('path');
+const { URL } = require('url');
+const { realpathSync } = require('fs');
+const preserveSymlinks = !!process.binding('config').preserveSymlinks;
+const {
+  ModuleWrap,
+  createDynamicModule
+} = require('internal/loader/ModuleWrap');
+const errors = require('internal/errors');
+
+const search = require('internal/loader/search');
+const asyncReadFile = require('util').promisify(require('fs').readFile);
+const debug = require('util').debuglog('esm');
+
+const realpathCache = new Map();
+
+const loaders = new Map();
+exports.loaders = loaders;
+
+// Strategy for loading a standard JavaScript module
+loaders.set('esm', async (url) => {
+  const source = `${await asyncReadFile(new URL(url))}`;
+  debug(`Loading StandardModule ${url}`);
+  return {
+    module: new ModuleWrap(internalCJSModule.stripShebang(source), url),
+    reflect: undefined
+  };
+});
+
+// Strategy for loading a node-style CommonJS module
+loaders.set('cjs', async (url) => {
+  return createDynamicModule(['default'], url, (reflect) => {
+    debug(`Loading CJSModule ${url}`);
+    const CJSModule = require('module');
+    const pathname = internalURLModule.getPathFromURL(new URL(url));
+    CJSModule._load(pathname);
+  });
+});
+
+// Strategy for loading a node builtin CommonJS module that isn't
+// through normal resolution
+loaders.set('builtin', async (url) => {
+  return createDynamicModule(['default'], url, (reflect) => {
+    debug(`Loading BuiltinModule ${url}`);
+    const exports = NativeModule.require(url.substr(5));
+    reflect.exports.default.set(exports);
+  });
+});
+
+loaders.set('addon', async (url) => {
+  const ctx = createDynamicModule(['default'], url, (reflect) => {
+    debug(`Loading NativeModule ${url}`);
+    const module = { exports: {} };
+    const pathname = internalURLModule.getPathFromURL(new URL(url));
+    process.dlopen(module, _makeLong(pathname));
+    reflect.exports.default.set(module.exports);
+  });
+  return ctx;
+});
+
+loaders.set('json', async (url) => {
+  return createDynamicModule(['default'], url, (reflect) => {
+    debug(`Loading JSONModule ${url}`);
+    const pathname = internalURLModule.getPathFromURL(new URL(url));
+    const content = fs.readFileSync(pathname, 'utf8');
+    try {
+      const exports = JSON.parse(internalCJSModule.stripBOM(content));
+      reflect.exports.default.set(exports);
+    } catch (err) {
+      err.message = pathname + ': ' + err.message;
+      throw err;
+    }
+  });
+});
+
+exports.resolve = (specifier, parentURL) => {
+  if (NativeModule.nonInternalExists(specifier)) {
+    return {
+      url: specifier,
+      format: 'builtin'
+    };
+  }
+
+  let url = search(specifier, parentURL);
+
+  if (url.protocol !== 'file:') {
+    throw new errors.Error('ERR_INVALID_PROTOCOL',
+                           url.protocol, 'file:');
+  }
+
+  if (!preserveSymlinks) {
+    const real = realpathSync(internalURLModule.getPathFromURL(url), {
+      [internalFS.realpathCacheKey]: realpathCache
+    });
+    const old = url;
+    url = internalURLModule.getURLFromFilePath(real);
+    url.search = old.search;
+    url.hash = old.hash;
+  }
+
+  const ext = extname(url.pathname);
+  switch (ext) {
+    case '.mjs':
+      return { url: `${url}`, format: 'esm' };
+    case '.json':
+      return { url: `${url}`, format: 'json' };
+    case '.node':
+      return { url: `${url}`, format: 'addon' };
+    case '.js':
+      return { url: `${url}`, format: 'cjs' };
+    default:
+      throw new errors.Error('ERR_UNKNOWN_FILE_EXTENSION',
+                             internalURLModule.getPathFromURL(url));
+  }
+};
diff --git a/lib/internal/loader/resolveRequestUrl.js b/lib/internal/loader/resolveRequestUrl.js
deleted file mode 100644
index 2245064bfe..0000000000
--- a/lib/internal/loader/resolveRequestUrl.js
+++ /dev/null
@@ -1,104 +0,0 @@
-'use strict';
-
-const { URL } = require('url');
-const internalCJSModule = require('internal/module');
-const internalURLModule = require('internal/url');
-const internalFS = require('internal/fs');
-const NativeModule = require('native_module');
-const { extname } = require('path');
-const { realpathSync } = require('fs');
-const preserveSymlinks = !!process.binding('config').preserveSymlinks;
-const {
-  ModuleWrap,
-  createDynamicModule
-} = require('internal/loader/ModuleWrap');
-const errors = require('internal/errors');
-
-const search = require('internal/loader/search');
-const asyncReadFile = require('util').promisify(require('fs').readFile);
-const debug = require('util').debuglog('esm');
-
-const realpathCache = new Map();
-
-class ModuleRequest {
-  constructor(url) {
-    this.url = url;
-  }
-}
-Object.setPrototypeOf(ModuleRequest.prototype, null);
-
-// Strategy for loading a standard JavaScript module
-class StandardModuleRequest extends ModuleRequest {
-  async createModule() {
-    const source = `${await asyncReadFile(this.url)}`;
-    debug(`Loading StandardModule ${this.url}`);
-    return new ModuleWrap(internalCJSModule.stripShebang(source),
-                          `${this.url}`);
-  }
-}
-
-// Strategy for loading a node-style CommonJS module
-class CJSModuleRequest extends ModuleRequest {
-  async createModule() {
-    const ctx = createDynamicModule(['default'], this.url, (reflect) => {
-      debug(`Loading CJSModule ${this.url.pathname}`);
-      const CJSModule = require('module');
-      const pathname = internalURLModule.getPathFromURL(this.url);
-      CJSModule._load(pathname);
-    });
-    this.finish = (module) => {
-      ctx.reflect.exports.default.set(module.exports);
-    };
-    return ctx.module;
-  }
-}
-
-// Strategy for loading a node builtin CommonJS module that isn't
-// through normal resolution
-class NativeModuleRequest extends CJSModuleRequest {
-  async createModule() {
-    const ctx = createDynamicModule(['default'], this.url, (reflect) => {
-      debug(`Loading NativeModule ${this.url.pathname}`);
-      const exports = require(this.url.pathname);
-      reflect.exports.default.set(exports);
-    });
-    return ctx.module;
-  }
-}
-
-const normalizeBaseURL = (baseURLOrString) => {
-  if (baseURLOrString instanceof URL) return baseURLOrString;
-  if (typeof baseURLOrString === 'string') return new URL(baseURLOrString);
-  return undefined;
-};
-
-const resolveRequestUrl = (baseURLOrString, specifier) => {
-  if (NativeModule.nonInternalExists(specifier)) {
-    return new NativeModuleRequest(new URL(`node:${specifier}`));
-  }
-
-  const baseURL = normalizeBaseURL(baseURLOrString);
-  let url = search(specifier, baseURL);
-
-  if (url.protocol !== 'file:') {
-    throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:');
-  }
-
-  if (!preserveSymlinks) {
-    const real = realpathSync(internalURLModule.getPathFromURL(url), {
-      [internalFS.realpathCacheKey]: realpathCache
-    });
-    const old = url;
-    url = internalURLModule.getURLFromFilePath(real);
-    url.search = old.search;
-    url.hash = old.hash;
-  }
-
-  const ext = extname(url.pathname);
-  if (ext === '.mjs') {
-    return new StandardModuleRequest(url);
-  }
-
-  return new CJSModuleRequest(url);
-};
-module.exports = resolveRequestUrl;
diff --git a/lib/internal/loader/search.js b/lib/internal/loader/search.js
index f0ec34ae4e..8b1a3e0f83 100644
--- a/lib/internal/loader/search.js
+++ b/lib/internal/loader/search.js
@@ -6,12 +6,10 @@ const errors = require('internal/errors');
 const { resolve } = process.binding('module_wrap');
 
 module.exports = (target, base) => {
-  target = `${target}`;
   if (base === undefined) {
     // We cannot search without a base.
     throw new errors.Error('ERR_MISSING_MODULE', target);
   }
-  base = `${base}`;
   try {
     return resolve(target, base);
   } catch (e) {
diff --git a/lib/module.js b/lib/module.js
index 6693bedeb1..73f3cc8dd8 100644
--- a/lib/module.js
+++ b/lib/module.js
@@ -40,7 +40,7 @@ const errors = require('internal/errors');
 const Loader = require('internal/loader/Loader');
 const ModuleJob = require('internal/loader/ModuleJob');
 const { createDynamicModule } = require('internal/loader/ModuleWrap');
-const ESMLoader = new Loader();
+let ESMLoader;
 
 function stat(filename) {
   filename = path.toNamespacedPath(filename);
@@ -424,29 +424,31 @@ Module._load = function(request, parent, isMain) {
   var filename = null;
 
   if (isMain) {
-    let err;
     try {
       filename = Module._resolveFilename(request, parent, isMain);
     } catch (e) {
       // try to keep stack
       e.stack;
-      err = e;
+      throw e;
     }
     if (experimentalModules) {
-      if (filename === null || /\.mjs$/.test(filename)) {
-        try {
-          ESMLoader.import(getURLFromFilePath(filename).href).catch((e) => {
-            console.error(e);
-            process.exit(1);
-          });
-          return;
-        } catch (e) {
-          // well, it isn't ESM
+      (async () => {
+        // loader setup
+        if (!ESMLoader) {
+          ESMLoader = new Loader();
+          const userLoader = process.binding('config').userLoader;
+          if (userLoader) {
+            const hooks = await new Loader().import(userLoader);
+            ESMLoader.hook(hooks);
+          }
         }
-      }
-    }
-    if (err) {
-      throw err;
+        await ESMLoader.import(getURLFromFilePath(filename).href);
+      })()
+      .catch((e) => {
+        console.error(e);
+        process.exit(1);
+      });
+      return;
     }
   } else {
     filename = Module._resolveFilename(request, parent, isMain);
@@ -521,16 +523,18 @@ Module.prototype.load = function(filename) {
   Module._extensions[extension](this, filename);
   this.loaded = true;
 
-  if (experimentalModules) {
+  if (ESMLoader) {
     const url = getURLFromFilePath(filename);
-    if (ESMLoader.moduleMap.has(`${url}`) !== true) {
+    const urlString = `${url}`;
+    if (ESMLoader.moduleMap.has(urlString) !== true) {
       const ctx = createDynamicModule(['default'], url);
       ctx.reflect.exports.default.set(this.exports);
-      ESMLoader.moduleMap.set(`${url}`,
-                              new ModuleJob(ESMLoader, ctx.module));
+      ESMLoader.moduleMap.set(urlString,
+                              new ModuleJob(ESMLoader, url, async () => ctx));
     } else {
-      ESMLoader.moduleMap.get(`${url}`).moduleProvider.finish(
-        Module._cache[filename]);
+      const job =  ESMLoader.moduleMap.get(urlString);
+      if (job.reflect)
+        job.reflect.exports.default.set(this.exports);
     }
   }
 };