module: Allow runMain to be ESM

This follows the EPS an allows the node CLI to have ESM as an entry point.
`node ./example.mjs`. A newer V8 is needed for `import()` so that is not
included. `import.meta` is still in specification stage so that also is not
included.

PR-URL: https://github.com/nodejs/node/pull/14369
Author: Bradley Farias <bradley.meck@gmail.com>
Author: Guy Bedford <guybedford@gmail.com>
Author: Jan Krems <jan.krems@groupon.com>
Author: Timothy Gu <timothygu99@gmail.com>
Author: Michaël Zasso <targos@protonmail.com>
Author: Anna Henningsen <anna@addaleax.net>
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Jeremiah Senkpiel <fishrock123@rocketmail.com>

10 files changed, 466 insertions, 0 deletions

diff --git a/lib/internal/bootstrap_node.js b/lib/internal/bootstrap_node.js
index 9d776674d2..b43a262682 100644
--- a/lib/internal/bootstrap_node.js
+++ b/lib/internal/bootstrap_node.js
@@ -109,6 +109,13 @@
         'DeprecationWarning', 'DEP0062', startup, true);
     }
 
+    if (process.binding('config').experimentalModules) {
+      process.emitWarning(
+        'The ESM module loader is experimental.',
+        'ExperimentalWarning', undefined);
+    }
+
+
     // There are various modes that Node can run in. The most common two
     // are running from a script and running the REPL - but there are a few
     // others like the debugger or running --eval arguments. Here we decide
diff --git a/lib/internal/errors.js b/lib/internal/errors.js
index 623e57d875..10e5be8a44 100644
--- a/lib/internal/errors.js
+++ b/lib/internal/errors.js
@@ -229,6 +229,9 @@ E('ERR_IPC_ONE_PIPE', 'Child process can have only one IPC pipe');
 E('ERR_IPC_SYNC_FORK', 'IPC cannot be used with synchronous forks');
 E('ERR_METHOD_NOT_IMPLEMENTED', 'The %s method is not implemented');
 E('ERR_MISSING_ARGS', missingArgs);
+E('ERR_MISSING_MODULE', 'Cannot find module %s');
+E('ERR_MODULE_RESOLUTION_LEGACY', '%s not found by import in %s.' +
+  'Legacy behavior in require would have found it at %s');
 E('ERR_MULTIPLE_CALLBACK', 'Callback called multiple times');
 E('ERR_NAPI_CONS_FUNCTION', 'Constructor must be a function');
 E('ERR_NAPI_CONS_PROTOTYPE_OBJECT', 'Constructor.prototype must be an object');
@@ -237,6 +240,7 @@ E('ERR_NO_ICU', '%s is not supported on Node.js compiled without ICU');
 E('ERR_NO_LONGER_SUPPORTED', '%s is no longer supported');
 E('ERR_OUTOFMEMORY', 'Out of memory');
 E('ERR_PARSE_HISTORY_DATA', 'Could not parse history data in %s');
+E('ERR_REQUIRE_ESM', 'Must use import to load ES Module: %s');
 E('ERR_SERVER_ALREADY_LISTEN',
   'Listen method has been called more than once without closing.');
 E('ERR_SOCKET_ALREADY_BOUND', 'Socket is already bound');
diff --git a/lib/internal/loader/Loader.js b/lib/internal/loader/Loader.js
new file mode 100644
index 0000000000..a409d397f8
--- /dev/null
+++ b/lib/internal/loader/Loader.js
@@ -0,0 +1,75 @@
+'use strict';
+
+const { URL } = require('url');
+const { getURLFromFilePath } = require('internal/url');
+
+const {
+  getNamespaceOfModuleWrap
+} = require('internal/loader/ModuleWrap');
+
+const ModuleMap = require('internal/loader/ModuleMap');
+const ModuleJob = require('internal/loader/ModuleJob');
+const resolveRequestUrl = require('internal/loader/resolveRequestUrl');
+const errors = require('internal/errors');
+
+function getBase() {
+  try {
+    return getURLFromFilePath(`${process.cwd()}/`);
+  } catch (e) {
+    e.stack;
+    // If the current working directory no longer exists.
+    if (e.code === 'ENOENT') {
+      return undefined;
+    }
+    throw e;
+  }
+}
+
+class Loader {
+  constructor(base = getBase()) {
+    this.moduleMap = new ModuleMap();
+    if (typeof base !== 'undefined' && base instanceof URL !== true) {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'base', 'URL');
+    }
+    this.base = base;
+  }
+
+  async resolve(specifier) {
+    const request = resolveRequestUrl(this.base, specifier);
+    if (request.url.protocol !== 'file:') {
+      throw new errors.Error('ERR_INVALID_PROTOCOL',
+                             request.url.protocol, 'file:');
+    }
+    return request.url;
+  }
+
+  async getModuleJob(dependentJob, specifier) {
+    if (!this.moduleMap.has(dependentJob.url)) {
+      throw new errors.Error('ERR_MISSING_MODULE', dependentJob.url);
+    }
+    const request = await resolveRequestUrl(dependentJob.url, specifier);
+    const url = `${request.url}`;
+    if (this.moduleMap.has(url)) {
+      return this.moduleMap.get(url);
+    }
+    const dependencyJob = new ModuleJob(this, request);
+    this.moduleMap.set(url, dependencyJob);
+    return dependencyJob;
+  }
+
+  async import(specifier) {
+    const request = await resolveRequestUrl(this.base, specifier);
+    const url = `${request.url}`;
+    let job;
+    if (this.moduleMap.has(url)) {
+      job = this.moduleMap.get(url);
+    } else {
+      job = new ModuleJob(this, request);
+      this.moduleMap.set(url, job);
+    }
+    const module = await job.run();
+    return getNamespaceOfModuleWrap(module);
+  }
+}
+Object.setPrototypeOf(Loader.prototype, null);
+module.exports = Loader;
diff --git a/lib/internal/loader/ModuleJob.js b/lib/internal/loader/ModuleJob.js
new file mode 100644
index 0000000000..db4cb6ae5c
--- /dev/null
+++ b/lib/internal/loader/ModuleJob.js
@@ -0,0 +1,116 @@
+'use strict';
+
+const { SafeSet, SafePromise } = require('internal/safe_globals');
+const resolvedPromise = SafePromise.resolve();
+const resolvedArrayPromise = SafePromise.resolve([]);
+const { ModuleWrap } = require('internal/loader/ModuleWrap');
+
+const NOOP = () => { /* No-op */ };
+class ModuleJob {
+  /**
+   * @param {module: ModuleWrap?, compiled: Promise} moduleProvider
+   */
+  constructor(loader, moduleProvider, url) {
+    this.url = `${moduleProvider.url}`;
+    this.moduleProvider = moduleProvider;
+    this.loader = loader;
+    this.error = null;
+    this.hadError = false;
+
+    if (moduleProvider instanceof ModuleWrap !== true) {
+      // linked == promise for dependency jobs, with module populated,
+      // module wrapper linked
+      this.modulePromise = this.moduleProvider.createModule();
+      this.module = undefined;
+      const linked = async () => {
+        const dependencyJobs = [];
+        this.module = await this.modulePromise;
+        this.module.link(async (dependencySpecifier) => {
+          const dependencyJobPromise =
+            this.loader.getModuleJob(this, dependencySpecifier);
+          dependencyJobs.push(dependencyJobPromise);
+          const dependencyJob = await dependencyJobPromise;
+          return dependencyJob.modulePromise;
+        });
+        return SafePromise.all(dependencyJobs);
+      };
+      this.linked = linked();
+
+      // instantiated == deep dependency jobs wrappers instantiated,
+      //module wrapper instantiated
+      this.instantiated = undefined;
+    } else {
+      const getModuleProvider = async () => moduleProvider;
+      this.modulePromise = getModuleProvider();
+      this.moduleProvider = { finish: NOOP };
+      this.module = moduleProvider;
+      this.linked = resolvedArrayPromise;
+      this.instantiated = this.modulePromise;
+    }
+  }
+
+  instantiate() {
+    if (this.instantiated) {
+      return this.instantiated;
+    }
+    return this.instantiated = new Promise(async (resolve, reject) => {
+      const jobsInGraph = new SafeSet();
+      let jobsReadyToInstantiate = 0;
+      // (this must be sync for counter to work)
+      const queueJob = (moduleJob) => {
+        if (jobsInGraph.has(moduleJob)) {
+          return;
+        }
+        jobsInGraph.add(moduleJob);
+        moduleJob.linked.then((dependencyJobs) => {
+          for (const dependencyJob of dependencyJobs) {
+            queueJob(dependencyJob);
+          }
+          checkComplete();
+        }, (e) => {
+          if (!this.hadError) {
+            this.error = e;
+            this.hadError = true;
+          }
+          checkComplete();
+        });
+      };
+      const checkComplete = () => {
+        if (++jobsReadyToInstantiate === jobsInGraph.size) {
+          // I believe we only throw once the whole tree is finished loading?
+          // or should the error bail early, leaving entire tree to still load?
+          if (this.hadError) {
+            reject(this.error);
+          } else {
+            try {
+              this.module.instantiate();
+              for (const dependencyJob of jobsInGraph) {
+                dependencyJob.instantiated = resolvedPromise;
+              }
+              resolve(this.module);
+            } catch (e) {
+              e.stack;
+              reject(e);
+            }
+          }
+        }
+      };
+      queueJob(this);
+    });
+  }
+
+  async run() {
+    const module = await this.instantiate();
+    try {
+      module.evaluate();
+    } catch (e) {
+      e.stack;
+      this.hadError = true;
+      this.error = e;
+      throw e;
+    }
+    return module;
+  }
+}
+Object.setPrototypeOf(ModuleJob.prototype, null);
+module.exports = ModuleJob;
diff --git a/lib/internal/loader/ModuleMap.js b/lib/internal/loader/ModuleMap.js
new file mode 100644
index 0000000000..aa238afbae
--- /dev/null
+++ b/lib/internal/loader/ModuleMap.js
@@ -0,0 +1,33 @@
+'use strict';
+
+const ModuleJob = require('internal/loader/ModuleJob');
+const { SafeMap } = require('internal/safe_globals');
+const debug = require('util').debuglog('esm');
+const errors = require('internal/errors');
+
+// Tracks the state of the loader-level module cache
+class ModuleMap extends SafeMap {
+  get(url) {
+    if (typeof url !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string');
+    }
+    return super.get(url);
+  }
+  set(url, job) {
+    if (typeof url !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string');
+    }
+    if (job instanceof ModuleJob !== true) {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'job', 'ModuleJob');
+    }
+    debug(`Storing ${url} in ModuleMap`);
+    return super.set(url, job);
+  }
+  has(url) {
+    if (typeof url !== 'string') {
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'url', 'string');
+    }
+    return super.has(url);
+  }
+}
+module.exports = ModuleMap;
diff --git a/lib/internal/loader/ModuleWrap.js b/lib/internal/loader/ModuleWrap.js
new file mode 100644
index 0000000000..4d35356ec2
--- /dev/null
+++ b/lib/internal/loader/ModuleWrap.js
@@ -0,0 +1,61 @@
+'use strict';
+
+const { ModuleWrap } = process.binding('module_wrap');
+const debug = require('util').debuglog('esm');
+const ArrayJoin = Function.call.bind(Array.prototype.join);
+const ArrayMap = Function.call.bind(Array.prototype.map);
+
+const getNamespaceOfModuleWrap = (m) => {
+  const tmp = new ModuleWrap('import * as _ from "";_;', '');
+  tmp.link(async () => m);
+  tmp.instantiate();
+  return tmp.evaluate();
+};
+
+const createDynamicModule = (exports, url = '', evaluate) => {
+  debug(
+    `creating ESM facade for ${url} with exports: ${ArrayJoin(exports, ', ')}`
+  );
+  const names = ArrayMap(exports, (name) => `${name}`);
+  // sanitized ESM for reflection purposes
+  const src = `export let executor;
+  ${ArrayJoin(ArrayMap(names, (name) => `export let $${name}`), ';\n')}
+  ;(() => [
+    fn => executor = fn,
+    { exports: { ${
+  ArrayJoin(ArrayMap(names, (name) => `${name}: {
+        get: () => $${name},
+        set: v => $${name} = v
+      }`), ',\n')
+} } }
+  ]);
+  `;
+  const reflectiveModule = new ModuleWrap(src, `cjs-facade:${url}`);
+  reflectiveModule.instantiate();
+  const [setExecutor, reflect] = reflectiveModule.evaluate()();
+  // public exposed ESM
+  const reexports = `import { executor,
+    ${ArrayMap(names, (name) => `$${name}`)}
+  } from "";
+  export {
+    ${ArrayJoin(ArrayMap(names, (name) => `$${name} as ${name}`), ', ')}
+  }
+  // add await to this later if top level await comes along
+  typeof executor === "function" ? executor() : void 0;`;
+  if (typeof evaluate === 'function') {
+    setExecutor(() => evaluate(reflect));
+  }
+  const runner = new ModuleWrap(reexports, `${url}`);
+  runner.link(async () => reflectiveModule);
+  runner.instantiate();
+  return {
+    module: runner,
+    reflect
+  };
+};
+
+module.exports = {
+  createDynamicModule,
+  getNamespaceOfModuleWrap,
+  ModuleWrap
+};
diff --git a/lib/internal/loader/resolveRequestUrl.js b/lib/internal/loader/resolveRequestUrl.js
new file mode 100644
index 0000000000..2245064bfe
--- /dev/null
+++ b/lib/internal/loader/resolveRequestUrl.js
@@ -0,0 +1,104 @@
+'use strict';
+
+const { URL } = require('url');
+const internalCJSModule = require('internal/module');
+const internalURLModule = require('internal/url');
+const internalFS = require('internal/fs');
+const NativeModule = require('native_module');
+const { extname } = require('path');
+const { realpathSync } = require('fs');
+const preserveSymlinks = !!process.binding('config').preserveSymlinks;
+const {
+  ModuleWrap,
+  createDynamicModule
+} = require('internal/loader/ModuleWrap');
+const errors = require('internal/errors');
+
+const search = require('internal/loader/search');
+const asyncReadFile = require('util').promisify(require('fs').readFile);
+const debug = require('util').debuglog('esm');
+
+const realpathCache = new Map();
+
+class ModuleRequest {
+  constructor(url) {
+    this.url = url;
+  }
+}
+Object.setPrototypeOf(ModuleRequest.prototype, null);
+
+// Strategy for loading a standard JavaScript module
+class StandardModuleRequest extends ModuleRequest {
+  async createModule() {
+    const source = `${await asyncReadFile(this.url)}`;
+    debug(`Loading StandardModule ${this.url}`);
+    return new ModuleWrap(internalCJSModule.stripShebang(source),
+                          `${this.url}`);
+  }
+}
+
+// Strategy for loading a node-style CommonJS module
+class CJSModuleRequest extends ModuleRequest {
+  async createModule() {
+    const ctx = createDynamicModule(['default'], this.url, (reflect) => {
+      debug(`Loading CJSModule ${this.url.pathname}`);
+      const CJSModule = require('module');
+      const pathname = internalURLModule.getPathFromURL(this.url);
+      CJSModule._load(pathname);
+    });
+    this.finish = (module) => {
+      ctx.reflect.exports.default.set(module.exports);
+    };
+    return ctx.module;
+  }
+}
+
+// Strategy for loading a node builtin CommonJS module that isn't
+// through normal resolution
+class NativeModuleRequest extends CJSModuleRequest {
+  async createModule() {
+    const ctx = createDynamicModule(['default'], this.url, (reflect) => {
+      debug(`Loading NativeModule ${this.url.pathname}`);
+      const exports = require(this.url.pathname);
+      reflect.exports.default.set(exports);
+    });
+    return ctx.module;
+  }
+}
+
+const normalizeBaseURL = (baseURLOrString) => {
+  if (baseURLOrString instanceof URL) return baseURLOrString;
+  if (typeof baseURLOrString === 'string') return new URL(baseURLOrString);
+  return undefined;
+};
+
+const resolveRequestUrl = (baseURLOrString, specifier) => {
+  if (NativeModule.nonInternalExists(specifier)) {
+    return new NativeModuleRequest(new URL(`node:${specifier}`));
+  }
+
+  const baseURL = normalizeBaseURL(baseURLOrString);
+  let url = search(specifier, baseURL);
+
+  if (url.protocol !== 'file:') {
+    throw new errors.Error('ERR_INVALID_PROTOCOL', url.protocol, 'file:');
+  }
+
+  if (!preserveSymlinks) {
+    const real = realpathSync(internalURLModule.getPathFromURL(url), {
+      [internalFS.realpathCacheKey]: realpathCache
+    });
+    const old = url;
+    url = internalURLModule.getURLFromFilePath(real);
+    url.search = old.search;
+    url.hash = old.hash;
+  }
+
+  const ext = extname(url.pathname);
+  if (ext === '.mjs') {
+    return new StandardModuleRequest(url);
+  }
+
+  return new CJSModuleRequest(url);
+};
+module.exports = resolveRequestUrl;
diff --git a/lib/internal/loader/search.js b/lib/internal/loader/search.js
new file mode 100644
index 0000000000..f0ec34ae4e
--- /dev/null
+++ b/lib/internal/loader/search.js
@@ -0,0 +1,33 @@
+'use strict';
+
+const { URL } = require('url');
+const CJSmodule = require('module');
+const errors = require('internal/errors');
+const { resolve } = process.binding('module_wrap');
+
+module.exports = (target, base) => {
+  target = `${target}`;
+  if (base === undefined) {
+    // We cannot search without a base.
+    throw new errors.Error('ERR_MISSING_MODULE', target);
+  }
+  base = `${base}`;
+  try {
+    return resolve(target, base);
+  } catch (e) {
+    e.stack; // cause V8 to generate stack before rethrow
+    let error = e;
+    try {
+      const questionedBase = new URL(base);
+      const tmpMod = new CJSmodule(questionedBase.pathname, null);
+      tmpMod.paths = CJSmodule._nodeModulePaths(
+        new URL('./', questionedBase).pathname);
+      const found = CJSmodule._resolveFilename(target, tmpMod);
+      error = new errors.Error('ERR_MODULE_RESOLUTION_LEGACY', target,
+                               base, found);
+    } catch (problemChecking) {
+      // ignore
+    }
+    throw error;
+  }
+};
diff --git a/lib/internal/safe_globals.js b/lib/internal/safe_globals.js
new file mode 100644
index 0000000000..ad58fa662b
--- /dev/null
+++ b/lib/internal/safe_globals.js
@@ -0,0 +1,26 @@
+'use strict';
+
+const copyProps = (unsafe, safe) => {
+  for (const key of [...Object.getOwnPropertyNames(unsafe),
+                     ...Object.getOwnPropertySymbols(unsafe)
+  ]) {
+    if (!Object.getOwnPropertyDescriptor(safe, key)) {
+      Object.defineProperty(
+        safe,
+        key,
+        Object.getOwnPropertyDescriptor(unsafe, key));
+    }
+  }
+};
+const makeSafe = (unsafe, safe) => {
+  copyProps(unsafe.prototype, safe.prototype);
+  copyProps(unsafe, safe);
+  Object.setPrototypeOf(safe.prototype, null);
+  Object.freeze(safe.prototype);
+  Object.freeze(safe);
+  return safe;
+};
+
+exports.SafeMap = makeSafe(Map, class SafeMap extends Map {});
+exports.SafeSet = makeSafe(Set, class SafeSet extends Set {});
+exports.SafePromise = makeSafe(Promise, class SafePromise extends Promise {});
diff --git a/lib/internal/url.js b/lib/internal/url.js
index 95b5e9c5fd..cf0271691a 100644
--- a/lib/internal/url.js
+++ b/lib/internal/url.js
@@ -1377,6 +1377,12 @@ function getPathFromURL(path) {
   return isWindows ? getPathFromURLWin32(path) : getPathFromURLPosix(path);
 }
 
+function getURLFromFilePath(filepath) {
+  const tmp = new URL('file://');
+  tmp.pathname = filepath;
+  return tmp;
+}
+
 function NativeURL(ctx) {
   this[context] = ctx;
 }
@@ -1405,6 +1411,7 @@ setURLConstructor(constructUrl);
 module.exports = {
   toUSVString,
   getPathFromURL,
+  getURLFromFilePath,
   URL,
   URLSearchParams,
   domainToASCII,