1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
|
// ## Token types
// The assignment of fine-grained, information-carrying type objects
// allows the tokenizer to store the information it has about a
// token in a way that is very cheap for the parser to look up.
// All token type variables start with an underscore, to make them
// easy to recognize.
// The `beforeExpr` property is used to disambiguate between regular
// expressions and divisions. It is set on all token types that can
// be followed by an expression (thus, a slash after them would be a
// regular expression).
//
// The `startsExpr` property is used to check if the token ends a
// `yield` expression. It is set on all token types that either can
// directly start an expression (like a quotation mark) or can
// continue an expression (like the body of a string).
//
// `isLoop` marks a keyword as starting a loop, which is important
// to know when parsing a label, in order to allow or disallow
// continue jumps to that label.
export class TokenType {
constructor(label, conf = {}) {
this.label = label
this.keyword = conf.keyword
this.beforeExpr = !!conf.beforeExpr
this.startsExpr = !!conf.startsExpr
this.isLoop = !!conf.isLoop
this.isAssign = !!conf.isAssign
this.prefix = !!conf.prefix
this.postfix = !!conf.postfix
this.binop = conf.binop || null
this.updateContext = null
}
}
function binop(name, prec) {
return new TokenType(name, {beforeExpr: true, binop: prec})
}
const beforeExpr = {beforeExpr: true}, startsExpr = {startsExpr: true}
// Map keyword names to token types.
export const keywords = {}
// Succinct definitions of keyword token types
function kw(name, options = {}) {
options.keyword = name
return keywords[name] = new TokenType(name, options)
}
export const types = {
num: new TokenType("num", startsExpr),
regexp: new TokenType("regexp", startsExpr),
string: new TokenType("string", startsExpr),
name: new TokenType("name", startsExpr),
eof: new TokenType("eof"),
// Punctuation token types.
bracketL: new TokenType("[", {beforeExpr: true, startsExpr: true}),
bracketR: new TokenType("]"),
braceL: new TokenType("{", {beforeExpr: true, startsExpr: true}),
braceR: new TokenType("}"),
parenL: new TokenType("(", {beforeExpr: true, startsExpr: true}),
parenR: new TokenType(")"),
comma: new TokenType(",", beforeExpr),
semi: new TokenType(";", beforeExpr),
colon: new TokenType(":", beforeExpr),
dot: new TokenType("."),
question: new TokenType("?", beforeExpr),
arrow: new TokenType("=>", beforeExpr),
template: new TokenType("template"),
ellipsis: new TokenType("...", beforeExpr),
backQuote: new TokenType("`", startsExpr),
dollarBraceL: new TokenType("${", {beforeExpr: true, startsExpr: true}),
// Operators. These carry several kinds of properties to help the
// parser use them properly (the presence of these properties is
// what categorizes them as operators).
//
// `binop`, when present, specifies that this operator is a binary
// operator, and will refer to its precedence.
//
// `prefix` and `postfix` mark the operator as a prefix or postfix
// unary operator.
//
// `isAssign` marks all of `=`, `+=`, `-=` etcetera, which act as
// binary operators with a very low precedence, that should result
// in AssignmentExpression nodes.
eq: new TokenType("=", {beforeExpr: true, isAssign: true}),
assign: new TokenType("_=", {beforeExpr: true, isAssign: true}),
incDec: new TokenType("++/--", {prefix: true, postfix: true, startsExpr: true}),
prefix: new TokenType("prefix", {beforeExpr: true, prefix: true, startsExpr: true}),
logicalOR: binop("||", 1),
logicalAND: binop("&&", 2),
bitwiseOR: binop("|", 3),
bitwiseXOR: binop("^", 4),
bitwiseAND: binop("&", 5),
equality: binop("==/!=", 6),
relational: binop("</>", 7),
bitShift: binop("<</>>", 8),
plusMin: new TokenType("+/-", {beforeExpr: true, binop: 9, prefix: true, startsExpr: true}),
modulo: binop("%", 10),
star: binop("*", 10),
slash: binop("/", 10),
starstar: new TokenType("**", {beforeExpr: true}),
// Keyword token types.
_break: kw("break"),
_case: kw("case", beforeExpr),
_catch: kw("catch"),
_continue: kw("continue"),
_debugger: kw("debugger"),
_default: kw("default", beforeExpr),
_do: kw("do", {isLoop: true, beforeExpr: true}),
_else: kw("else", beforeExpr),
_finally: kw("finally"),
_for: kw("for", {isLoop: true}),
_function: kw("function", startsExpr),
_if: kw("if"),
_return: kw("return", beforeExpr),
_switch: kw("switch"),
_throw: kw("throw", beforeExpr),
_try: kw("try"),
_var: kw("var"),
_const: kw("const"),
_while: kw("while", {isLoop: true}),
_with: kw("with"),
_new: kw("new", {beforeExpr: true, startsExpr: true}),
_this: kw("this", startsExpr),
_super: kw("super", startsExpr),
_class: kw("class"),
_extends: kw("extends", beforeExpr),
_export: kw("export"),
_import: kw("import"),
_null: kw("null", startsExpr),
_true: kw("true", startsExpr),
_false: kw("false", startsExpr),
_in: kw("in", {beforeExpr: true, binop: 7}),
_instanceof: kw("instanceof", {beforeExpr: true, binop: 7}),
_typeof: kw("typeof", {beforeExpr: true, prefix: true, startsExpr: true}),
_void: kw("void", {beforeExpr: true, prefix: true, startsExpr: true}),
_delete: kw("delete", {beforeExpr: true, prefix: true, startsExpr: true})
}
|
