fs: introduce `opendir()` and `fs.Dir`

This adds long-requested methods for asynchronously interacting and
iterating through directory entries by using `uv_fs_opendir`,
`uv_fs_readdir`, and `uv_fs_closedir`.

`fs.opendir()` and friends return an `fs.Dir`, which contains methods
for doing reads and cleanup. `fs.Dir` also has the async iterator
symbol exposed.

The `read()` method and friends only return `fs.Dirent`s for this API.
Having a entry type or doing a `stat` call is deemed to be necessary in
the majority of cases, so just returning dirents seems like the logical
choice for a new api.

Reading when there are no more entries returns `null` instead of a
dirent. However the async iterator hides that (and does automatic
cleanup).

The code lives in separate files from the rest of fs, this is done
partially to prevent over-pollution of those (already very large)
files, but also in the case of js allows loading into `fsPromises`.

Due to async_hooks, this introduces a new handle type of `DIRHANDLE`.

This PR does not attempt to make complete optimization of
this feature. Notable future improvements include:
- Moving promise work into C++ land like FileHandle.
- Possibly adding `readv()` to do multi-entry directory reads.
- Aliasing `fs.readdir` to `fs.scandir` and doing a deprecation.

Refs: https://github.com/nodejs/node-v0.x-archive/issues/388
Refs: https://github.com/nodejs/node/issues/583
Refs: https://github.com/libuv/libuv/pull/2057

PR-URL: https://github.com/nodejs/node/pull/29349
Reviewed-By: Anna Henningsen <anna@addaleax.net>
Reviewed-By: David Carlier <devnexen@gmail.com>

5 files changed, 256 insertions, 27 deletions

diff --git a/lib/fs.js b/lib/fs.js
index 5c7d907f5e..34517a17da 100644
--- a/lib/fs.js
+++ b/lib/fs.js
@@ -64,6 +64,7 @@ const {
   getDirents,
   getOptions,
   getValidatedPath,
+  handleErrorFromBinding,
   nullCheck,
   preprocessSymlinkDestination,
   Stats,
@@ -80,6 +81,11 @@ const {
   warnOnNonPortableTemplate
 } = require('internal/fs/utils');
 const {
+  Dir,
+  opendir,
+  opendirSync
+} = require('internal/fs/dir');
+const {
   CHAR_FORWARD_SLASH,
   CHAR_BACKWARD_SLASH,
 } = require('internal/constants');
@@ -122,23 +128,6 @@ function showTruncateDeprecation() {
   }
 }
 
-function handleErrorFromBinding(ctx) {
-  if (ctx.errno !== undefined) {  // libuv error numbers
-    const err = uvException(ctx);
-    // eslint-disable-next-line no-restricted-syntax
-    Error.captureStackTrace(err, handleErrorFromBinding);
-    throw err;
-  }
-  if (ctx.error !== undefined) {  // Errors created in C++ land.
-    // TODO(joyeecheung): currently, ctx.error are encoding errors
-    // usually caused by memory problems. We need to figure out proper error
-    // code(s) for this.
-    // eslint-disable-next-line no-restricted-syntax
-    Error.captureStackTrace(ctx.error, handleErrorFromBinding);
-    throw ctx.error;
-  }
-}
-
 function maybeCallback(cb) {
   if (typeof cb === 'function')
     return cb;
@@ -1834,7 +1823,6 @@ function createWriteStream(path, options) {
   return new WriteStream(path, options);
 }
 
-
 module.exports = fs = {
   appendFile,
   appendFileSync,
@@ -1880,6 +1868,8 @@ module.exports = fs = {
   mkdtempSync,
   open,
   openSync,
+  opendir,
+  opendirSync,
   readdir,
   readdirSync,
   read,
@@ -1913,6 +1903,7 @@ module.exports = fs = {
   writeSync,
   writev,
   writevSync,
+  Dir,
   Dirent,
   Stats,
 
diff --git a/lib/internal/errors.js b/lib/internal/errors.js
index 39e6b6deda..cd3c162183 100644
--- a/lib/internal/errors.js
+++ b/lib/internal/errors.js
@@ -764,6 +764,7 @@ E('ERR_CRYPTO_SCRYPT_NOT_SUPPORTED', 'Scrypt algorithm not supported', Error);
 E('ERR_CRYPTO_SIGN_KEY_REQUIRED', 'No key provided to sign', Error);
 E('ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH',
   'Input buffers must have the same byte length', RangeError);
+E('ERR_DIR_CLOSED', 'Directory handle was closed', Error);
 E('ERR_DNS_SET_SERVERS_FAILED', 'c-ares failed to set servers: "%s" [%s]',
   Error);
 E('ERR_DOMAIN_CALLBACK_NOT_AVAILABLE',
diff --git a/lib/internal/fs/dir.js b/lib/internal/fs/dir.js
new file mode 100644
index 0000000000..c1a25f0de3
--- /dev/null
+++ b/lib/internal/fs/dir.js
@@ -0,0 +1,201 @@
+'use strict';
+
+const { Object } = primordials;
+
+const pathModule = require('path');
+const binding = internalBinding('fs');
+const dirBinding = internalBinding('fs_dir');
+const {
+  codes: {
+    ERR_DIR_CLOSED,
+    ERR_INVALID_CALLBACK,
+    ERR_MISSING_ARGS
+  }
+} = require('internal/errors');
+
+const { FSReqCallback } = binding;
+const internalUtil = require('internal/util');
+const {
+  getDirent,
+  getOptions,
+  getValidatedPath,
+  handleErrorFromBinding
+} = require('internal/fs/utils');
+
+const kDirHandle = Symbol('kDirHandle');
+const kDirPath = Symbol('kDirPath');
+const kDirClosed = Symbol('kDirClosed');
+const kDirOptions = Symbol('kDirOptions');
+const kDirReadPromisified = Symbol('kDirReadPromisified');
+const kDirClosePromisified = Symbol('kDirClosePromisified');
+
+class Dir {
+  constructor(handle, path, options) {
+    if (handle == null) throw new ERR_MISSING_ARGS('handle');
+    this[kDirHandle] = handle;
+    this[kDirPath] = path;
+    this[kDirClosed] = false;
+
+    this[kDirOptions] = getOptions(options, {
+      encoding: 'utf8'
+    });
+
+    this[kDirReadPromisified] = internalUtil.promisify(this.read).bind(this);
+    this[kDirClosePromisified] = internalUtil.promisify(this.close).bind(this);
+  }
+
+  get path() {
+    return this[kDirPath];
+  }
+
+  read(options, callback) {
+    if (this[kDirClosed] === true) {
+      throw new ERR_DIR_CLOSED();
+    }
+
+    callback = typeof options === 'function' ? options : callback;
+    if (callback === undefined) {
+      return this[kDirReadPromisified](options);
+    } else if (typeof callback !== 'function') {
+      throw new ERR_INVALID_CALLBACK(callback);
+    }
+    options = getOptions(options, this[kDirOptions]);
+
+    const req = new FSReqCallback();
+    req.oncomplete = (err, result) => {
+      if (err || result === null) {
+        return callback(err, result);
+      }
+      getDirent(this[kDirPath], result[0], result[1], callback);
+    };
+
+    this[kDirHandle].read(
+      options.encoding,
+      req
+    );
+  }
+
+  readSync(options) {
+    if (this[kDirClosed] === true) {
+      throw new ERR_DIR_CLOSED();
+    }
+
+    options = getOptions(options, this[kDirOptions]);
+
+    const ctx = { path: this[kDirPath] };
+    const result = this[kDirHandle].read(
+      options.encoding,
+      undefined,
+      ctx
+    );
+    handleErrorFromBinding(ctx);
+
+    if (result === null) {
+      return result;
+    }
+
+    return getDirent(this[kDirPath], result[0], result[1]);
+  }
+
+  close(callback) {
+    if (this[kDirClosed] === true) {
+      throw new ERR_DIR_CLOSED();
+    }
+
+    if (callback === undefined) {
+      return this[kDirClosePromisified]();
+    } else if (typeof callback !== 'function') {
+      throw new ERR_INVALID_CALLBACK(callback);
+    }
+
+    this[kDirClosed] = true;
+    const req = new FSReqCallback();
+    req.oncomplete = callback;
+    this[kDirHandle].close(req);
+  }
+
+  closeSync() {
+    if (this[kDirClosed] === true) {
+      throw new ERR_DIR_CLOSED();
+    }
+
+    this[kDirClosed] = true;
+    const ctx = { path: this[kDirPath] };
+    const result = this[kDirHandle].close(undefined, ctx);
+    handleErrorFromBinding(ctx);
+    return result;
+  }
+
+  async* entries() {
+    try {
+      while (true) {
+        const result = await this[kDirReadPromisified]();
+        if (result === null) {
+          break;
+        }
+        yield result;
+      }
+    } finally {
+      await this[kDirClosePromisified]();
+    }
+  }
+}
+
+Object.defineProperty(Dir.prototype, Symbol.asyncIterator, {
+  value: Dir.prototype.entries,
+  enumerable: false,
+  writable: true,
+  configurable: true,
+});
+
+function opendir(path, options, callback) {
+  callback = typeof options === 'function' ? options : callback;
+  if (typeof callback !== 'function') {
+    throw new ERR_INVALID_CALLBACK(callback);
+  }
+  path = getValidatedPath(path);
+  options = getOptions(options, {
+    encoding: 'utf8'
+  });
+
+  function opendirCallback(error, handle) {
+    if (error) {
+      callback(error);
+    } else {
+      callback(null, new Dir(handle, path, options));
+    }
+  }
+
+  const req = new FSReqCallback();
+  req.oncomplete = opendirCallback;
+
+  dirBinding.opendir(
+    pathModule.toNamespacedPath(path),
+    options.encoding,
+    req
+  );
+}
+
+function opendirSync(path, options) {
+  path = getValidatedPath(path);
+  options = getOptions(options, {
+    encoding: 'utf8'
+  });
+
+  const ctx = { path };
+  const handle = dirBinding.opendir(
+    pathModule.toNamespacedPath(path),
+    options.encoding,
+    undefined,
+    ctx
+  );
+  handleErrorFromBinding(ctx);
+
+  return new Dir(handle, path, options);
+}
+
+module.exports = {
+  Dir,
+  opendir,
+  opendirSync
+};
diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js
index 5af2ffa763..7660ff66be 100644
--- a/lib/internal/fs/promises.js
+++ b/lib/internal/fs/promises.js
@@ -36,6 +36,7 @@ const {
   validateRmdirOptions,
   warnOnNonPortableTemplate
 } = require('internal/fs/utils');
+const { opendir } = require('internal/fs/dir');
 const {
   parseMode,
   validateBuffer,
@@ -509,6 +510,7 @@ module.exports = {
   access,
   copyFile,
   open,
+  opendir: promisify(opendir),
   rename,
   truncate,
   rmdir,
diff --git a/lib/internal/fs/utils.js b/lib/internal/fs/utils.js
index f6050f53ee..3324ec5080 100644
--- a/lib/internal/fs/utils.js
+++ b/lib/internal/fs/utils.js
@@ -12,7 +12,8 @@ const {
     ERR_INVALID_OPT_VALUE_ENCODING,
     ERR_OUT_OF_RANGE
   },
-  hideStackFrames
+  hideStackFrames,
+  uvException
 } = require('internal/errors');
 const {
   isArrayBufferView,
@@ -165,19 +166,33 @@ function getDirents(path, [names, types], callback) {
   } else {
     const len = names.length;
     for (i = 0; i < len; i++) {
-      const type = types[i];
-      if (type === UV_DIRENT_UNKNOWN) {
-        const name = names[i];
-        const stats = lazyLoadFs().lstatSync(pathModule.join(path, name));
-        names[i] = new DirentFromStats(name, stats);
-      } else {
-        names[i] = new Dirent(names[i], types[i]);
-      }
+      names[i] = getDirent(path, names[i], types[i]);
     }
     return names;
   }
 }
 
+function getDirent(path, name, type, callback) {
+  if (typeof callback === 'function') {
+    if (type === UV_DIRENT_UNKNOWN) {
+      lazyLoadFs().lstat(pathModule.join(path, name), (err, stats) => {
+        if (err) {
+          callback(err);
+          return;
+        }
+        callback(null, new DirentFromStats(name, stats));
+      });
+    } else {
+      callback(null, new Dirent(name, type));
+    }
+  } else if (type === UV_DIRENT_UNKNOWN) {
+    const stats = lazyLoadFs().lstatSync(pathModule.join(path, name));
+    return new DirentFromStats(name, stats);
+  } else {
+    return new Dirent(name, type);
+  }
+}
+
 function getOptions(options, defaultOptions) {
   if (options === null || options === undefined ||
       typeof options === 'function') {
@@ -197,6 +212,23 @@ function getOptions(options, defaultOptions) {
   return options;
 }
 
+function handleErrorFromBinding(ctx) {
+  if (ctx.errno !== undefined) {  // libuv error numbers
+    const err = uvException(ctx);
+    // eslint-disable-next-line no-restricted-syntax
+    Error.captureStackTrace(err, handleErrorFromBinding);
+    throw err;
+  }
+  if (ctx.error !== undefined) {  // Errors created in C++ land.
+    // TODO(joyeecheung): currently, ctx.error are encoding errors
+    // usually caused by memory problems. We need to figure out proper error
+    // code(s) for this.
+    // eslint-disable-next-line no-restricted-syntax
+    Error.captureStackTrace(ctx.error, handleErrorFromBinding);
+    throw ctx.error;
+  }
+}
+
 // Check if the path contains null types if it is a string nor Uint8Array,
 // otherwise return silently.
 const nullCheck = hideStackFrames((path, propName, throwError = true) => {
@@ -558,9 +590,11 @@ module.exports = {
   BigIntStats,  // for testing
   copyObject,
   Dirent,
+  getDirent,
   getDirents,
   getOptions,
   getValidatedPath,
+  handleErrorFromBinding,
   nullCheck,
   preprocessSymlinkDestination,
   realpathCacheKey: Symbol('realpathCacheKey'),