/** * @fileoverview Object to handle access and retrieval of tokens. * @author Brandon Mills */ "use strict"; //------------------------------------------------------------------------------ // Requirements //------------------------------------------------------------------------------ const assert = require("assert"); const { isCommentToken } = require("eslint-utils"); const cursors = require("./cursors"); const ForwardTokenCursor = require("./forward-token-cursor"); const PaddedTokenCursor = require("./padded-token-cursor"); const utils = require("./utils"); //------------------------------------------------------------------------------ // Helpers //------------------------------------------------------------------------------ const TOKENS = Symbol("tokens"); const COMMENTS = Symbol("comments"); const INDEX_MAP = Symbol("indexMap"); /** * Creates the map from locations to indices in `tokens`. * * The first/last location of tokens is mapped to the index of the token. * The first/last location of comments is mapped to the index of the next token of each comment. * @param {Token[]} tokens The array of tokens. * @param {Comment[]} comments The array of comments. * @returns {Object} The map from locations to indices in `tokens`. * @private */ function createIndexMap(tokens, comments) { const map = Object.create(null); let tokenIndex = 0; let commentIndex = 0; let nextStart = 0; let range = null; while (tokenIndex < tokens.length || commentIndex < comments.length) { nextStart = (commentIndex < comments.length) ? comments[commentIndex].range[0] : Number.MAX_SAFE_INTEGER; while (tokenIndex < tokens.length && (range = tokens[tokenIndex].range)[0] < nextStart) { map[range[0]] = tokenIndex; map[range[1] - 1] = tokenIndex; tokenIndex += 1; } nextStart = (tokenIndex < tokens.length) ? tokens[tokenIndex].range[0] : Number.MAX_SAFE_INTEGER; while (commentIndex < comments.length && (range = comments[commentIndex].range)[0] < nextStart) { map[range[0]] = tokenIndex; map[range[1] - 1] = tokenIndex; commentIndex += 1; } } return map; } /** * Creates the cursor iterates tokens with options. * @param {CursorFactory} factory The cursor factory to initialize cursor. * @param {Token[]} tokens The array of tokens. * @param {Comment[]} comments The array of comments. * @param {Object} indexMap The map from locations to indices in `tokens`. * @param {number} startLoc The start location of the iteration range. * @param {number} endLoc The end location of the iteration range. * @param {number|Function|Object} [opts=0] The option object. If this is a number then it's `opts.skip`. If this is a function then it's `opts.filter`. * @param {boolean} [opts.includeComments=false] The flag to iterate comments as well. * @param {Function|null} [opts.filter=null] The predicate function to choose tokens. * @param {number} [opts.skip=0] The count of tokens the cursor skips. * @returns {Cursor} The created cursor. * @private */ function createCursorWithSkip(factory, tokens, comments, indexMap, startLoc, endLoc, opts) { let includeComments = false; let skip = 0; let filter = null; if (typeof opts === "number") { skip = opts | 0; } else if (typeof opts === "function") { filter = opts; } else if (opts) { includeComments = !!opts.includeComments; skip = opts.skip | 0; filter = opts.filter || null; } assert(skip >= 0, "options.skip should be zero or a positive integer."); assert(!filter || typeof filter === "function", "options.filter should be a function."); return factory.createCursor(tokens, comments, indexMap, startLoc, endLoc, includeComments, filter, skip, -1); } /** * Creates the cursor iterates tokens with options. * @param {CursorFactory} factory The cursor factory to initialize cursor. * @param {Token[]} tokens The array of tokens. * @param {Comment[]} comments The array of comments. * @param {Object} indexMap The map from locations to indices in `tokens`. * @param {number} startLoc The start location of the iteration range. * @param {number} endLoc The end location of the iteration range. * @param {number|Function|Object} [opts=0] The option object. If this is a number then it's `opts.count`. If this is a function then it's `opts.filter`. * @param {boolean} [opts.includeComments] The flag to iterate comments as well. * @param {Function|null} [opts.filter=null] The predicate function to choose tokens. * @param {number} [opts.count=0] The maximum count of tokens the cursor iterates. Zero is no iteration for backward compatibility. * @returns {Cursor} The created cursor. * @private */ function createCursorWithCount(factory, tokens, comments, indexMap, startLoc, endLoc, opts) { let includeComments = false; let count = 0; let countExists = false; let filter = null; if (typeof opts === "number") { count = opts | 0; countExists = true; } else if (typeof opts === "function") { filter = opts; } else if (opts) { includeComments = !!opts.includeComments; count = opts.count | 0; countExists = typeof opts.count === "number"; filter = opts.filter || null; } assert(count >= 0, "options.count should be zero or a positive integer."); assert(!filter || typeof filter === "function", "options.filter should be a function."); return factory.createCursor(tokens, comments, indexMap, startLoc, endLoc, includeComments, filter, 0, countExists ? count : -1); } /** * Creates the cursor iterates tokens with options. * This is overload function of the below. * @param {Token[]} tokens The array of tokens. * @param {Comment[]} comments The array of comments. * @param {Object} indexMap The map from locations to indices in `tokens`. * @param {number} startLoc The start location of the iteration range. * @param {number} endLoc The end location of the iteration range. * @param {Function|Object} opts The option object. If this is a function then it's `opts.filter`. * @param {boolean} [opts.includeComments] The flag to iterate comments as well. * @param {Function|null} [opts.filter=null] The predicate function to choose tokens. * @param {number} [opts.count=0] The maximum count of tokens the cursor iterates. Zero is no iteration for backward compatibility. * @returns {Cursor} The created cursor. * @private */ /** * Creates the cursor iterates tokens with options. * @param {Token[]} tokens The array of tokens. * @param {Comment[]} comments The array of comments. * @param {Object} indexMap The map from locations to indices in `tokens`. * @param {number} startLoc The start location of the iteration range. * @param {number} endLoc The end location of the iteration range. * @param {number} [beforeCount=0] The number of tokens before the node to retrieve. * @param {boolean} [afterCount=0] The number of tokens after the node to retrieve. * @returns {Cursor} The created cursor. * @private */ function createCursorWithPadding(tokens, comments, indexMap, startLoc, endLoc, beforeCount, afterCount) { if (typeof beforeCount === "undefined" && typeof afterCount === "undefined") { return new ForwardTokenCursor(tokens, comments, indexMap, startLoc, endLoc); } if (typeof beforeCount === "number" || typeof beforeCount === "undefined") { return new PaddedTokenCursor(tokens, comments, indexMap, startLoc, endLoc, beforeCount | 0, afterCount | 0); } return createCursorWithCount(cursors.forward, tokens, comments, indexMap, startLoc, endLoc, beforeCount); } /** * Gets comment tokens that are adjacent to the current cursor position. * @param {Cursor} cursor A cursor instance. * @returns {Array} An array of comment tokens adjacent to the current cursor position. * @private */ function getAdjacentCommentTokensFromCursor(cursor) { const tokens = []; let currentToken = cursor.getOneToken(); while (currentToken && isCommentToken(currentToken)) { tokens.push(currentToken); currentToken = cursor.getOneToken(); } return tokens; } //------------------------------------------------------------------------------ // Exports //------------------------------------------------------------------------------ /** * The token store. * * This class provides methods to get tokens by locations as fast as possible. * The methods are a part of public API, so we should be careful if it changes this class. * * People can get tokens in O(1) by the hash map which is mapping from the location of tokens/comments to tokens. * Also people can get a mix of tokens and comments in O(log k), the k is the number of comments. * Assuming that comments to be much fewer than tokens, this does not make hash map from token's locations to comments to reduce memory cost. * This uses binary-searching instead for comments. */ module.exports = class TokenStore { /** * Initializes this token store. * @param {Token[]} tokens The array of tokens. * @param {Comment[]} comments The array of comments. */ constructor(tokens, comments) { this[TOKENS] = tokens; this[COMMENTS] = comments; this[INDEX_MAP] = createIndexMap(tokens, comments); } //-------------------------------------------------------------------------- // Gets single token. //-------------------------------------------------------------------------- /** * Gets the token starting at the specified index. * @param {number} offset Index of the start of the token's range. * @param {Object} [options=0] The option object. * @param {boolean} [options.includeComments=false] The flag to iterate comments as well. * @returns {Token|null} The token starting at index, or null if no such token. */ getTokenByRangeStart(offset, options) { const includeComments = options && options.includeComments; const token = cursors.forward.createBaseCursor( this[TOKENS], this[COMMENTS], this[INDEX_MAP], offset, -1, includeComments ).getOneToken(); if (token && token.range[0] === offset) { return token; } return null; } /** * Gets the first token of the given node. * @param {ASTNode} node The AST node. * @param {number|Function|Object} [options=0] The option object. If this is a number then it's `options.skip`. If this is a function then it's `options.filter`. * @param {boolean} [options.includeComments=false] The flag to iterate comments as well. * @param {Function|null} [options.filter=null] The predicate function to choose tokens. * @param {number} [options.skip=0] The count of tokens the cursor skips. * @returns {Token|null} An object representing the token. */ getFirstToken(node, options) { return createCursorWithSkip( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[0], node.range[1], options ).getOneToken(); } /** * Gets the last token of the given node. * @param {ASTNode} node The AST node. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstToken() * @returns {Token|null} An object representing the token. */ getLastToken(node, options) { return createCursorWithSkip( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[0], node.range[1], options ).getOneToken(); } /** * Gets the token that precedes a given node or token. * @param {ASTNode|Token|Comment} node The AST node or token. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstToken() * @returns {Token|null} An object representing the token. */ getTokenBefore(node, options) { return createCursorWithSkip( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], -1, node.range[0], options ).getOneToken(); } /** * Gets the token that follows a given node or token. * @param {ASTNode|Token|Comment} node The AST node or token. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstToken() * @returns {Token|null} An object representing the token. */ getTokenAfter(node, options) { return createCursorWithSkip( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[1], -1, options ).getOneToken(); } /** * Gets the first token between two non-overlapping nodes. * @param {ASTNode|Token|Comment} left Node before the desired token range. * @param {ASTNode|Token|Comment} right Node after the desired token range. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstToken() * @returns {Token|null} An object representing the token. */ getFirstTokenBetween(left, right, options) { return createCursorWithSkip( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], left.range[1], right.range[0], options ).getOneToken(); } /** * Gets the last token between two non-overlapping nodes. * @param {ASTNode|Token|Comment} left Node before the desired token range. * @param {ASTNode|Token|Comment} right Node after the desired token range. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstToken() * @returns {Token|null} An object representing the token. */ getLastTokenBetween(left, right, options) { return createCursorWithSkip( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], left.range[1], right.range[0], options ).getOneToken(); } /** * Gets the token that precedes a given node or token in the token stream. * This is defined for backward compatibility. Use `includeComments` option instead. * TODO: We have a plan to remove this in a future major version. * @param {ASTNode|Token|Comment} node The AST node or token. * @param {number} [skip=0] A number of tokens to skip. * @returns {Token|null} An object representing the token. * @deprecated */ getTokenOrCommentBefore(node, skip) { return this.getTokenBefore(node, { includeComments: true, skip }); } /** * Gets the token that follows a given node or token in the token stream. * This is defined for backward compatibility. Use `includeComments` option instead. * TODO: We have a plan to remove this in a future major version. * @param {ASTNode|Token|Comment} node The AST node or token. * @param {number} [skip=0] A number of tokens to skip. * @returns {Token|null} An object representing the token. * @deprecated */ getTokenOrCommentAfter(node, skip) { return this.getTokenAfter(node, { includeComments: true, skip }); } //-------------------------------------------------------------------------- // Gets multiple tokens. //-------------------------------------------------------------------------- /** * Gets the first `count` tokens of the given node. * @param {ASTNode} node The AST node. * @param {number|Function|Object} [options=0] The option object. If this is a number then it's `options.count`. If this is a function then it's `options.filter`. * @param {boolean} [options.includeComments=false] The flag to iterate comments as well. * @param {Function|null} [options.filter=null] The predicate function to choose tokens. * @param {number} [options.count=0] The maximum count of tokens the cursor iterates. * @returns {Token[]} Tokens. */ getFirstTokens(node, options) { return createCursorWithCount( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[0], node.range[1], options ).getAllTokens(); } /** * Gets the last `count` tokens of the given node. * @param {ASTNode} node The AST node. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstTokens() * @returns {Token[]} Tokens. */ getLastTokens(node, options) { return createCursorWithCount( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[0], node.range[1], options ).getAllTokens().reverse(); } /** * Gets the `count` tokens that precedes a given node or token. * @param {ASTNode|Token|Comment} node The AST node or token. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstTokens() * @returns {Token[]} Tokens. */ getTokensBefore(node, options) { return createCursorWithCount( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], -1, node.range[0], options ).getAllTokens().reverse(); } /** * Gets the `count` tokens that follows a given node or token. * @param {ASTNode|Token|Comment} node The AST node or token. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstTokens() * @returns {Token[]} Tokens. */ getTokensAfter(node, options) { return createCursorWithCount( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[1], -1, options ).getAllTokens(); } /** * Gets the first `count` tokens between two non-overlapping nodes. * @param {ASTNode|Token|Comment} left Node before the desired token range. * @param {ASTNode|Token|Comment} right Node after the desired token range. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstTokens() * @returns {Token[]} Tokens between left and right. */ getFirstTokensBetween(left, right, options) { return createCursorWithCount( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], left.range[1], right.range[0], options ).getAllTokens(); } /** * Gets the last `count` tokens between two non-overlapping nodes. * @param {ASTNode|Token|Comment} left Node before the desired token range. * @param {ASTNode|Token|Comment} right Node after the desired token range. * @param {number|Function|Object} [options=0] The option object. Same options as getFirstTokens() * @returns {Token[]} Tokens between left and right. */ getLastTokensBetween(left, right, options) { return createCursorWithCount( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], left.range[1], right.range[0], options ).getAllTokens().reverse(); } /** * Gets all tokens that are related to the given node. * @param {ASTNode} node The AST node. * @param {Function|Object} options The option object. If this is a function then it's `options.filter`. * @param {boolean} [options.includeComments=false] The flag to iterate comments as well. * @param {Function|null} [options.filter=null] The predicate function to choose tokens. * @param {number} [options.count=0] The maximum count of tokens the cursor iterates. * @returns {Token[]} Array of objects representing tokens. */ /** * Gets all tokens that are related to the given node. * @param {ASTNode} node The AST node. * @param {int} [beforeCount=0] The number of tokens before the node to retrieve. * @param {int} [afterCount=0] The number of tokens after the node to retrieve. * @returns {Token[]} Array of objects representing tokens. */ getTokens(node, beforeCount, afterCount) { return createCursorWithPadding( this[TOKENS], this[COMMENTS], this[INDEX_MAP], node.range[0], node.range[1], beforeCount, afterCount ).getAllTokens(); } /** * Gets all of the tokens between two non-overlapping nodes. * @param {ASTNode|Token|Comment} left Node before the desired token range. * @param {ASTNode|Token|Comment} right Node after the desired token range. * @param {Function|Object} options The option object. If this is a function then it's `options.filter`. * @param {boolean} [options.includeComments=false] The flag to iterate comments as well. * @param {Function|null} [options.filter=null] The predicate function to choose tokens. * @param {number} [options.count=0] The maximum count of tokens the cursor iterates. * @returns {Token[]} Tokens between left and right. */ /** * Gets all of the tokens between two non-overlapping nodes. * @param {ASTNode|Token|Comment} left Node before the desired token range. * @param {ASTNode|Token|Comment} right Node after the desired token range. * @param {int} [padding=0] Number of extra tokens on either side of center. * @returns {Token[]} Tokens between left and right. */ getTokensBetween(left, right, padding) { return createCursorWithPadding( this[TOKENS], this[COMMENTS], this[INDEX_MAP], left.range[1], right.range[0], padding, padding ).getAllTokens(); } //-------------------------------------------------------------------------- // Others. //-------------------------------------------------------------------------- /** * Checks whether any comments exist or not between the given 2 nodes. * @param {ASTNode} left The node to check. * @param {ASTNode} right The node to check. * @returns {boolean} `true` if one or more comments exist. */ commentsExistBetween(left, right) { const index = utils.search(this[COMMENTS], left.range[1]); return ( index < this[COMMENTS].length && this[COMMENTS][index].range[1] <= right.range[0] ); } /** * Gets all comment tokens directly before the given node or token. * @param {ASTNode|token} nodeOrToken The AST node or token to check for adjacent comment tokens. * @returns {Array} An array of comments in occurrence order. */ getCommentsBefore(nodeOrToken) { const cursor = createCursorWithCount( cursors.backward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], -1, nodeOrToken.range[0], { includeComments: true } ); return getAdjacentCommentTokensFromCursor(cursor).reverse(); } /** * Gets all comment tokens directly after the given node or token. * @param {ASTNode|token} nodeOrToken The AST node or token to check for adjacent comment tokens. * @returns {Array} An array of comments in occurrence order. */ getCommentsAfter(nodeOrToken) { const cursor = createCursorWithCount( cursors.forward, this[TOKENS], this[COMMENTS], this[INDEX_MAP], nodeOrToken.range[1], -1, { includeComments: true } ); return getAdjacentCommentTokensFromCursor(cursor); } /** * Gets all comment tokens inside the given node. * @param {ASTNode} node The AST node to get the comments for. * @returns {Array} An array of comments in occurrence order. */ getCommentsInside(node) { return this.getTokens(node, { includeComments: true, filter: isCommentToken }); } };