diff options
Diffstat (limited to 'deps/v8/src/parsing/token.h')
-rw-r--r-- | deps/v8/src/parsing/token.h | 192 |
1 files changed, 106 insertions, 86 deletions
diff --git a/deps/v8/src/parsing/token.h b/deps/v8/src/parsing/token.h index e1c6239e36..c457d39e92 100644 --- a/deps/v8/src/parsing/token.h +++ b/deps/v8/src/parsing/token.h @@ -20,12 +20,6 @@ namespace internal { // // T: Non-keyword tokens // K: Keyword tokens -// C: Contextual keyword token -// -// Contextual keyword tokens are tokens that are scanned as Token::IDENTIFIER, -// but that in some contexts are treated as keywords. This mostly happens -// when ECMAScript introduces new keywords, but for backwards compatibility -// allows them to still be used as indentifiers in most contexts. // IGNORE_TOKEN is a convenience macro that can be supplied as // an argument (at any position) for a TOKEN_LIST call. It does @@ -33,7 +27,8 @@ namespace internal { #define IGNORE_TOKEN(name, string, precedence) -/* Binary operators sorted by precedence */ +/* Binary operators */ +/* ADD and SUB are at the end since they are UnaryOp */ #define BINARY_OP_TOKEN_LIST(T, E) \ E(T, BIT_OR, "|", 6) \ E(T, BIT_XOR, "^", 7) \ @@ -41,12 +36,12 @@ namespace internal { E(T, SHL, "<<", 11) \ E(T, SAR, ">>", 11) \ E(T, SHR, ">>>", 11) \ - E(T, ADD, "+", 12) \ - E(T, SUB, "-", 12) \ E(T, MUL, "*", 13) \ E(T, DIV, "/", 13) \ E(T, MOD, "%", 13) \ - E(T, EXP, "**", 14) + E(T, EXP, "**", 14) \ + E(T, ADD, "+", 12) \ + E(T, SUB, "-", 12) #define EXPAND_BINOP_ASSIGN_TOKEN(T, name, string, precedence) \ T(ASSIGN_##name, string "=", 2) @@ -54,32 +49,47 @@ namespace internal { #define EXPAND_BINOP_TOKEN(T, name, string, precedence) \ T(name, string, precedence) -#define TOKEN_LIST(T, K, C) \ - /* End of source indicator. */ \ - T(EOS, "EOS", 0) \ +#define TOKEN_LIST(T, K) \ + \ + /* BEGIN PropertyOrCall */ \ + /* BEGIN Member */ \ + /* BEGIN Template */ \ + /* ES6 Template Literals */ \ + T(TEMPLATE_SPAN, nullptr, 0) \ + T(TEMPLATE_TAIL, nullptr, 0) \ + /* END Template */ \ \ /* Punctuators (ECMA-262, section 7.7, page 15). */ \ + /* BEGIN Property */ \ + T(PERIOD, ".", 0) \ + T(LBRACK, "[", 0) \ + /* END Property */ \ + /* END Member */ \ T(LPAREN, "(", 0) \ + /* END PropertyOrCall */ \ T(RPAREN, ")", 0) \ - T(LBRACK, "[", 0) \ T(RBRACK, "]", 0) \ T(LBRACE, "{", 0) \ - T(RBRACE, "}", 0) \ T(COLON, ":", 0) \ - T(SEMICOLON, ";", 0) \ - T(PERIOD, ".", 0) \ T(ELLIPSIS, "...", 0) \ T(CONDITIONAL, "?", 3) \ - T(INC, "++", 0) \ - T(DEC, "--", 0) \ - T(ARROW, "=>", 0) \ + /* BEGIN AutoSemicolon */ \ + T(SEMICOLON, ";", 0) \ + T(RBRACE, "}", 0) \ + /* End of source indicator. */ \ + T(EOS, "EOS", 0) \ + /* END AutoSemicolon */ \ \ - /* Assignment operators. */ \ + /* BEGIN ArrowOrAssignmentOp */ \ + T(ARROW, "=>", 0) \ + /* BEGIN AssignmentOp */ \ /* IsAssignmentOp() relies on this block of enum values being */ \ /* contiguous and sorted in the same order! */ \ T(INIT, "=init", 2) /* AST-use only. */ \ T(ASSIGN, "=", 2) \ BINARY_OP_TOKEN_LIST(T, EXPAND_BINOP_ASSIGN_TOKEN) \ + /* END AssignmentOp */ \ + /* END ArrowOrAssignmentOp */ \ \ /* Binary operators sorted by precedence. */ \ /* IsBinaryOp() relies on this block of enum values */ \ @@ -87,8 +97,24 @@ namespace internal { T(COMMA, ",", 1) \ T(OR, "||", 4) \ T(AND, "&&", 5) \ + \ + /* Unary operators, starting at ADD in BINARY_OP_TOKEN_LIST */ \ + /* IsUnaryOp() relies on this block of enum values */ \ + /* being contiguous and sorted in the same order! */ \ BINARY_OP_TOKEN_LIST(T, EXPAND_BINOP_TOKEN) \ \ + T(NOT, "!", 0) \ + T(BIT_NOT, "~", 0) \ + K(DELETE, "delete", 0) \ + K(TYPEOF, "typeof", 0) \ + K(VOID, "void", 0) \ + \ + /* BEGIN IsCountOp */ \ + T(INC, "++", 0) \ + T(DEC, "--", 0) \ + /* END IsCountOp */ \ + /* END IsUnaryOrCountOp */ \ + \ /* Compare operators sorted by precedence. */ \ /* IsCompareOp() relies on this block of enum values */ \ /* being contiguous and sorted in the same order! */ \ @@ -103,15 +129,6 @@ namespace internal { K(INSTANCEOF, "instanceof", 10) \ K(IN, "in", 10) \ \ - /* Unary operators. */ \ - /* IsUnaryOp() relies on this block of enum values */ \ - /* being contiguous and sorted in the same order! */ \ - T(NOT, "!", 0) \ - T(BIT_NOT, "~", 0) \ - K(DELETE, "delete", 0) \ - K(TYPEOF, "typeof", 0) \ - K(VOID, "void", 0) \ - \ /* Keywords (ECMA-262, section 7.5.2, page 13). */ \ K(BREAK, "break", 0) \ K(CASE, "case", 0) \ @@ -149,6 +166,8 @@ namespace internal { T(BIGINT, nullptr, 0) \ T(STRING, nullptr, 0) \ \ + /* BEGIN Callable */ \ + K(SUPER, "super", 0) \ /* BEGIN AnyIdentifier */ \ /* Identifiers (not keywords or future reserved words). */ \ T(IDENTIFIER, nullptr, 0) \ @@ -161,14 +180,14 @@ namespace internal { /* Future reserved words (ECMA-262, section 7.6.1.2). */ \ T(FUTURE_STRICT_RESERVED_WORD, nullptr, 0) \ T(ESCAPED_STRICT_RESERVED_WORD, nullptr, 0) \ - K(ENUM, "enum", 0) \ /* END AnyIdentifier */ \ + /* END Callable */ \ + K(ENUM, "enum", 0) \ K(CLASS, "class", 0) \ K(CONST, "const", 0) \ K(EXPORT, "export", 0) \ K(EXTENDS, "extends", 0) \ K(IMPORT, "import", 0) \ - K(SUPER, "super", 0) \ T(PRIVATE_NAME, nullptr, 0) \ \ /* Illegal token - not able to scan. */ \ @@ -178,35 +197,13 @@ namespace internal { /* Scanner-internal use only. */ \ T(WHITESPACE, nullptr, 0) \ T(UNINITIALIZED, nullptr, 0) \ - T(REGEXP_LITERAL, nullptr, 0) \ - \ - /* ES6 Template Literals */ \ - T(TEMPLATE_SPAN, nullptr, 0) \ - T(TEMPLATE_TAIL, nullptr, 0) \ - \ - /* Contextual keyword tokens */ \ - C(GET, "get", 0) \ - C(SET, "set", 0) \ - C(OF, "of", 0) \ - C(TARGET, "target", 0) \ - C(META, "meta", 0) \ - C(AS, "as", 0) \ - C(FROM, "from", 0) \ - C(NAME, "name", 0) \ - C(PROTO_UNDERSCORED, "__proto__", 0) \ - C(CONSTRUCTOR, "constructor", 0) \ - C(PRIVATE_CONSTRUCTOR, "#constructor", 0) \ - C(PROTOTYPE, "prototype", 0) \ - C(EVAL, "eval", 0) \ - C(ARGUMENTS, "arguments", 0) \ - C(UNDEFINED, "undefined", 0) \ - C(ANONYMOUS, "anonymous", 0) + T(REGEXP_LITERAL, nullptr, 0) class Token { public: // All token values. #define T(name, string, precedence) name, - enum Value : uint8_t { TOKEN_LIST(T, T, T) NUM_TOKENS }; + enum Value : uint8_t { TOKEN_LIST(T, T) NUM_TOKENS }; #undef T // Returns a string corresponding to the C++ token name @@ -216,43 +213,73 @@ class Token { return name_[token]; } - static char TypeForTesting(Value token) { return token_type[token]; } + class IsKeywordBits : public BitField8<bool, 0, 1> {}; + class IsPropertyNameBits : public BitField8<bool, IsKeywordBits::kNext, 1> {}; // Predicates - static bool IsKeyword(Value token) { return token_type[token] == 'K'; } - static bool IsContextualKeyword(Value token) { - return IsInRange(token, GET, ANONYMOUS); + static bool IsKeyword(Value token) { + return IsKeywordBits::decode(token_flags[token]); + } + + static bool IsPropertyName(Value token) { + return IsPropertyNameBits::decode(token_flags[token]); } - static bool IsIdentifier(Value token, LanguageMode language_mode, - bool is_generator, bool disallow_await) { - if (IsInRange(token, IDENTIFIER, ASYNC)) return true; - if (IsInRange(token, LET, ESCAPED_STRICT_RESERVED_WORD)) { - return is_sloppy(language_mode); - } + V8_INLINE static bool IsValidIdentifier(Value token, + LanguageMode language_mode, + bool is_generator, + bool disallow_await) { + if (V8_LIKELY(IsInRange(token, IDENTIFIER, ASYNC))) return true; if (token == AWAIT) return !disallow_await; if (token == YIELD) return !is_generator && is_sloppy(language_mode); - return false; + return IsStrictReservedWord(token) && is_sloppy(language_mode); + } + + static bool IsCallable(Value token) { + return IsInRange(token, SUPER, ESCAPED_STRICT_RESERVED_WORD); + } + + static bool IsAutoSemicolon(Value token) { + return IsInRange(token, SEMICOLON, EOS); } static bool IsAnyIdentifier(Value token) { - return IsInRange(token, IDENTIFIER, ENUM); + return IsInRange(token, IDENTIFIER, ESCAPED_STRICT_RESERVED_WORD); } static bool IsStrictReservedWord(Value token) { - return IsInRange(token, LET, ESCAPED_STRICT_RESERVED_WORD); + return IsInRange(token, YIELD, ESCAPED_STRICT_RESERVED_WORD); } static bool IsLiteral(Value token) { return IsInRange(token, NULL_LITERAL, STRING); } + static bool IsTemplate(Value token) { + return IsInRange(token, TEMPLATE_SPAN, TEMPLATE_TAIL); + } + + static bool IsMember(Value token) { + return IsInRange(token, TEMPLATE_SPAN, LBRACK); + } + + static bool IsProperty(Value token) { + return IsInRange(token, PERIOD, LBRACK); + } + + static bool IsPropertyOrCall(Value token) { + return IsInRange(token, TEMPLATE_SPAN, LPAREN); + } + + static bool IsArrowOrAssignmentOp(Value token) { + return IsInRange(token, ARROW, ASSIGN_SUB); + } + static bool IsAssignmentOp(Value token) { - return IsInRange(token, INIT, ASSIGN_EXP); + return IsInRange(token, INIT, ASSIGN_SUB); } - static bool IsGetOrSet(Value op) { return IsInRange(op, GET, SET); } - static bool IsBinaryOp(Value op) { return IsInRange(op, COMMA, EXP); } + static bool IsBinaryOp(Value op) { return IsInRange(op, COMMA, SUB); } static bool IsCompareOp(Value op) { return IsInRange(op, EQ, IN); } @@ -263,7 +290,7 @@ class Token { static bool IsEqualityOp(Value op) { return IsInRange(op, EQ, EQ_STRICT); } static Value BinaryOpForAssignment(Value op) { - DCHECK(IsInRange(op, ASSIGN_BIT_OR, ASSIGN_EXP)); + DCHECK(IsInRange(op, ASSIGN_BIT_OR, ASSIGN_SUB)); Value result = static_cast<Value>(op - ASSIGN_BIT_OR + BIT_OR); DCHECK(IsBinaryOp(result)); return result; @@ -273,18 +300,11 @@ class Token { return IsInRange(op, BIT_OR, SHR) || op == BIT_NOT; } - static bool IsUnaryOp(Value op) { - return IsInRange(op, NOT, VOID) || IsInRange(op, ADD, SUB); - } - + static bool IsUnaryOp(Value op) { return IsInRange(op, ADD, VOID); } static bool IsCountOp(Value op) { return IsInRange(op, INC, DEC); } - + static bool IsUnaryOrCountOp(Value op) { return IsInRange(op, ADD, DEC); } static bool IsShiftOp(Value op) { return IsInRange(op, SHL, SHR); } - static bool IsTrivialExpressionToken(Value op) { - return IsInRange(op, THIS, IDENTIFIER); - } - // Returns a string corresponding to the JS token string // (.e., "<" for the token LT) or nullptr if the token doesn't // have a (unique) string (e.g. an IDENTIFIER). @@ -300,17 +320,17 @@ class Token { // Returns the precedence > 0 for binary and compare // operators; returns 0 otherwise. - static int Precedence(Value token) { + static int Precedence(Value token, bool accept_IN) { DCHECK_GT(NUM_TOKENS, token); // token is unsigned - return precedence_[token]; + return precedence_[accept_IN][token]; } private: static const char* const name_[NUM_TOKENS]; static const char* const string_[NUM_TOKENS]; static const uint8_t string_length_[NUM_TOKENS]; - static const int8_t precedence_[NUM_TOKENS]; - static const char token_type[NUM_TOKENS]; + static const int8_t precedence_[2][NUM_TOKENS]; + static const uint8_t token_flags[NUM_TOKENS]; }; } // namespace internal |