From cbd8d715b2286e5726e6988921f5c870cbf74127 Mon Sep 17 00:00:00 2001 From: Jeremiah Senkpiel Date: Tue, 27 Aug 2019 17:14:27 -0700 Subject: fs: introduce `opendir()` and `fs.Dir` This adds long-requested methods for asynchronously interacting and iterating through directory entries by using `uv_fs_opendir`, `uv_fs_readdir`, and `uv_fs_closedir`. `fs.opendir()` and friends return an `fs.Dir`, which contains methods for doing reads and cleanup. `fs.Dir` also has the async iterator symbol exposed. The `read()` method and friends only return `fs.Dirent`s for this API. Having a entry type or doing a `stat` call is deemed to be necessary in the majority of cases, so just returning dirents seems like the logical choice for a new api. Reading when there are no more entries returns `null` instead of a dirent. However the async iterator hides that (and does automatic cleanup). The code lives in separate files from the rest of fs, this is done partially to prevent over-pollution of those (already very large) files, but also in the case of js allows loading into `fsPromises`. Due to async_hooks, this introduces a new handle type of `DIRHANDLE`. This PR does not attempt to make complete optimization of this feature. Notable future improvements include: - Moving promise work into C++ land like FileHandle. - Possibly adding `readv()` to do multi-entry directory reads. - Aliasing `fs.readdir` to `fs.scandir` and doing a deprecation. Refs: https://github.com/nodejs/node-v0.x-archive/issues/388 Refs: https://github.com/nodejs/node/issues/583 Refs: https://github.com/libuv/libuv/pull/2057 PR-URL: https://github.com/nodejs/node/pull/29349 Reviewed-By: Anna Henningsen Reviewed-By: David Carlier --- lib/fs.js | 27 ++---- lib/internal/errors.js | 1 + lib/internal/fs/dir.js | 201 ++++++++++++++++++++++++++++++++++++++++++++ lib/internal/fs/promises.js | 2 + lib/internal/fs/utils.js | 52 ++++++++++-- 5 files changed, 256 insertions(+), 27 deletions(-) create mode 100644 lib/internal/fs/dir.js (limited to 'lib') diff --git a/lib/fs.js b/lib/fs.js index 5c7d907f5e..34517a17da 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -64,6 +64,7 @@ const { getDirents, getOptions, getValidatedPath, + handleErrorFromBinding, nullCheck, preprocessSymlinkDestination, Stats, @@ -79,6 +80,11 @@ const { validateRmdirOptions, warnOnNonPortableTemplate } = require('internal/fs/utils'); +const { + Dir, + opendir, + opendirSync +} = require('internal/fs/dir'); const { CHAR_FORWARD_SLASH, CHAR_BACKWARD_SLASH, @@ -122,23 +128,6 @@ function showTruncateDeprecation() { } } -function handleErrorFromBinding(ctx) { - if (ctx.errno !== undefined) { // libuv error numbers - const err = uvException(ctx); - // eslint-disable-next-line no-restricted-syntax - Error.captureStackTrace(err, handleErrorFromBinding); - throw err; - } - if (ctx.error !== undefined) { // Errors created in C++ land. - // TODO(joyeecheung): currently, ctx.error are encoding errors - // usually caused by memory problems. We need to figure out proper error - // code(s) for this. - // eslint-disable-next-line no-restricted-syntax - Error.captureStackTrace(ctx.error, handleErrorFromBinding); - throw ctx.error; - } -} - function maybeCallback(cb) { if (typeof cb === 'function') return cb; @@ -1834,7 +1823,6 @@ function createWriteStream(path, options) { return new WriteStream(path, options); } - module.exports = fs = { appendFile, appendFileSync, @@ -1880,6 +1868,8 @@ module.exports = fs = { mkdtempSync, open, openSync, + opendir, + opendirSync, readdir, readdirSync, read, @@ -1913,6 +1903,7 @@ module.exports = fs = { writeSync, writev, writevSync, + Dir, Dirent, Stats, diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 39e6b6deda..cd3c162183 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -764,6 +764,7 @@ E('ERR_CRYPTO_SCRYPT_NOT_SUPPORTED', 'Scrypt algorithm not supported', Error); E('ERR_CRYPTO_SIGN_KEY_REQUIRED', 'No key provided to sign', Error); E('ERR_CRYPTO_TIMING_SAFE_EQUAL_LENGTH', 'Input buffers must have the same byte length', RangeError); +E('ERR_DIR_CLOSED', 'Directory handle was closed', Error); E('ERR_DNS_SET_SERVERS_FAILED', 'c-ares failed to set servers: "%s" [%s]', Error); E('ERR_DOMAIN_CALLBACK_NOT_AVAILABLE', diff --git a/lib/internal/fs/dir.js b/lib/internal/fs/dir.js new file mode 100644 index 0000000000..c1a25f0de3 --- /dev/null +++ b/lib/internal/fs/dir.js @@ -0,0 +1,201 @@ +'use strict'; + +const { Object } = primordials; + +const pathModule = require('path'); +const binding = internalBinding('fs'); +const dirBinding = internalBinding('fs_dir'); +const { + codes: { + ERR_DIR_CLOSED, + ERR_INVALID_CALLBACK, + ERR_MISSING_ARGS + } +} = require('internal/errors'); + +const { FSReqCallback } = binding; +const internalUtil = require('internal/util'); +const { + getDirent, + getOptions, + getValidatedPath, + handleErrorFromBinding +} = require('internal/fs/utils'); + +const kDirHandle = Symbol('kDirHandle'); +const kDirPath = Symbol('kDirPath'); +const kDirClosed = Symbol('kDirClosed'); +const kDirOptions = Symbol('kDirOptions'); +const kDirReadPromisified = Symbol('kDirReadPromisified'); +const kDirClosePromisified = Symbol('kDirClosePromisified'); + +class Dir { + constructor(handle, path, options) { + if (handle == null) throw new ERR_MISSING_ARGS('handle'); + this[kDirHandle] = handle; + this[kDirPath] = path; + this[kDirClosed] = false; + + this[kDirOptions] = getOptions(options, { + encoding: 'utf8' + }); + + this[kDirReadPromisified] = internalUtil.promisify(this.read).bind(this); + this[kDirClosePromisified] = internalUtil.promisify(this.close).bind(this); + } + + get path() { + return this[kDirPath]; + } + + read(options, callback) { + if (this[kDirClosed] === true) { + throw new ERR_DIR_CLOSED(); + } + + callback = typeof options === 'function' ? options : callback; + if (callback === undefined) { + return this[kDirReadPromisified](options); + } else if (typeof callback !== 'function') { + throw new ERR_INVALID_CALLBACK(callback); + } + options = getOptions(options, this[kDirOptions]); + + const req = new FSReqCallback(); + req.oncomplete = (err, result) => { + if (err || result === null) { + return callback(err, result); + } + getDirent(this[kDirPath], result[0], result[1], callback); + }; + + this[kDirHandle].read( + options.encoding, + req + ); + } + + readSync(options) { + if (this[kDirClosed] === true) { + throw new ERR_DIR_CLOSED(); + } + + options = getOptions(options, this[kDirOptions]); + + const ctx = { path: this[kDirPath] }; + const result = this[kDirHandle].read( + options.encoding, + undefined, + ctx + ); + handleErrorFromBinding(ctx); + + if (result === null) { + return result; + } + + return getDirent(this[kDirPath], result[0], result[1]); + } + + close(callback) { + if (this[kDirClosed] === true) { + throw new ERR_DIR_CLOSED(); + } + + if (callback === undefined) { + return this[kDirClosePromisified](); + } else if (typeof callback !== 'function') { + throw new ERR_INVALID_CALLBACK(callback); + } + + this[kDirClosed] = true; + const req = new FSReqCallback(); + req.oncomplete = callback; + this[kDirHandle].close(req); + } + + closeSync() { + if (this[kDirClosed] === true) { + throw new ERR_DIR_CLOSED(); + } + + this[kDirClosed] = true; + const ctx = { path: this[kDirPath] }; + const result = this[kDirHandle].close(undefined, ctx); + handleErrorFromBinding(ctx); + return result; + } + + async* entries() { + try { + while (true) { + const result = await this[kDirReadPromisified](); + if (result === null) { + break; + } + yield result; + } + } finally { + await this[kDirClosePromisified](); + } + } +} + +Object.defineProperty(Dir.prototype, Symbol.asyncIterator, { + value: Dir.prototype.entries, + enumerable: false, + writable: true, + configurable: true, +}); + +function opendir(path, options, callback) { + callback = typeof options === 'function' ? options : callback; + if (typeof callback !== 'function') { + throw new ERR_INVALID_CALLBACK(callback); + } + path = getValidatedPath(path); + options = getOptions(options, { + encoding: 'utf8' + }); + + function opendirCallback(error, handle) { + if (error) { + callback(error); + } else { + callback(null, new Dir(handle, path, options)); + } + } + + const req = new FSReqCallback(); + req.oncomplete = opendirCallback; + + dirBinding.opendir( + pathModule.toNamespacedPath(path), + options.encoding, + req + ); +} + +function opendirSync(path, options) { + path = getValidatedPath(path); + options = getOptions(options, { + encoding: 'utf8' + }); + + const ctx = { path }; + const handle = dirBinding.opendir( + pathModule.toNamespacedPath(path), + options.encoding, + undefined, + ctx + ); + handleErrorFromBinding(ctx); + + return new Dir(handle, path, options); +} + +module.exports = { + Dir, + opendir, + opendirSync +}; diff --git a/lib/internal/fs/promises.js b/lib/internal/fs/promises.js index 5af2ffa763..7660ff66be 100644 --- a/lib/internal/fs/promises.js +++ b/lib/internal/fs/promises.js @@ -36,6 +36,7 @@ const { validateRmdirOptions, warnOnNonPortableTemplate } = require('internal/fs/utils'); +const { opendir } = require('internal/fs/dir'); const { parseMode, validateBuffer, @@ -509,6 +510,7 @@ module.exports = { access, copyFile, open, + opendir: promisify(opendir), rename, truncate, rmdir, diff --git a/lib/internal/fs/utils.js b/lib/internal/fs/utils.js index f6050f53ee..3324ec5080 100644 --- a/lib/internal/fs/utils.js +++ b/lib/internal/fs/utils.js @@ -12,7 +12,8 @@ const { ERR_INVALID_OPT_VALUE_ENCODING, ERR_OUT_OF_RANGE }, - hideStackFrames + hideStackFrames, + uvException } = require('internal/errors'); const { isArrayBufferView, @@ -165,19 +166,33 @@ function getDirents(path, [names, types], callback) { } else { const len = names.length; for (i = 0; i < len; i++) { - const type = types[i]; - if (type === UV_DIRENT_UNKNOWN) { - const name = names[i]; - const stats = lazyLoadFs().lstatSync(pathModule.join(path, name)); - names[i] = new DirentFromStats(name, stats); - } else { - names[i] = new Dirent(names[i], types[i]); - } + names[i] = getDirent(path, names[i], types[i]); } return names; } } +function getDirent(path, name, type, callback) { + if (typeof callback === 'function') { + if (type === UV_DIRENT_UNKNOWN) { + lazyLoadFs().lstat(pathModule.join(path, name), (err, stats) => { + if (err) { + callback(err); + return; + } + callback(null, new DirentFromStats(name, stats)); + }); + } else { + callback(null, new Dirent(name, type)); + } + } else if (type === UV_DIRENT_UNKNOWN) { + const stats = lazyLoadFs().lstatSync(pathModule.join(path, name)); + return new DirentFromStats(name, stats); + } else { + return new Dirent(name, type); + } +} + function getOptions(options, defaultOptions) { if (options === null || options === undefined || typeof options === 'function') { @@ -197,6 +212,23 @@ function getOptions(options, defaultOptions) { return options; } +function handleErrorFromBinding(ctx) { + if (ctx.errno !== undefined) { // libuv error numbers + const err = uvException(ctx); + // eslint-disable-next-line no-restricted-syntax + Error.captureStackTrace(err, handleErrorFromBinding); + throw err; + } + if (ctx.error !== undefined) { // Errors created in C++ land. + // TODO(joyeecheung): currently, ctx.error are encoding errors + // usually caused by memory problems. We need to figure out proper error + // code(s) for this. + // eslint-disable-next-line no-restricted-syntax + Error.captureStackTrace(ctx.error, handleErrorFromBinding); + throw ctx.error; + } +} + // Check if the path contains null types if it is a string nor Uint8Array, // otherwise return silently. const nullCheck = hideStackFrames((path, propName, throwError = true) => { @@ -558,9 +590,11 @@ module.exports = { BigIntStats, // for testing copyObject, Dirent, + getDirent, getDirents, getOptions, getValidatedPath, + handleErrorFromBinding, nullCheck, preprocessSymlinkDestination, realpathCacheKey: Symbol('realpathCacheKey'), -- cgit v1.2.3