diff options
Diffstat (limited to 'deps/v8/src/scanner.h')
-rw-r--r-- | deps/v8/src/scanner.h | 64 |
1 files changed, 19 insertions, 45 deletions
diff --git a/deps/v8/src/scanner.h b/deps/v8/src/scanner.h index 92588905ad..a86ed07ab9 100644 --- a/deps/v8/src/scanner.h +++ b/deps/v8/src/scanner.h @@ -25,6 +25,7 @@ namespace internal { class AstRawString; class AstValueFactory; class ParserRecorder; +class UnicodeCache; // Returns the value (0 .. 15) of a hexadecimal character c. @@ -109,45 +110,6 @@ class Utf16CharacterStream { // --------------------------------------------------------------------- -// Caching predicates used by scanners. - -class UnicodeCache { - public: - UnicodeCache() {} - typedef unibrow::Utf8Decoder<512> Utf8Decoder; - - StaticResource<Utf8Decoder>* utf8_decoder() { - return &utf8_decoder_; - } - - bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } - bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } - bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } - bool IsLineTerminatorSequence(unibrow::uchar c, unibrow::uchar next) { - if (!IsLineTerminator(c)) return false; - if (c == 0x000d && next == 0x000a) return false; // CR with following LF. - return true; - } - - bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } - bool IsWhiteSpaceOrLineTerminator(unibrow::uchar c) { - return kIsWhiteSpaceOrLineTerminator.get(c); - } - - private: - unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; - unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; - unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; - unibrow::Predicate<WhiteSpace, 128> kIsWhiteSpace; - unibrow::Predicate<WhiteSpaceOrLineTerminator, 128> - kIsWhiteSpaceOrLineTerminator; - StaticResource<Utf8Decoder> utf8_decoder_; - - DISALLOW_COPY_AND_ASSIGN(UnicodeCache); -}; - - -// --------------------------------------------------------------------- // DuplicateFinder discovers duplicate symbols. class DuplicateFinder { @@ -399,6 +361,8 @@ class Scanner { // Returns the next token and advances input. Token::Value Next(); + // Returns the token following peek() + Token::Value PeekAhead(); // Returns the current token again. Token::Value current_token() { return current_.token; } // Returns the location information for the current token @@ -527,6 +491,7 @@ class Scanner { // Initialize current_ to not refer to a literal. current_.literal_chars = NULL; current_.raw_literal_chars = NULL; + next_next_.token = Token::UNINITIALIZED; } // Support BookmarkScope functionality. @@ -539,16 +504,22 @@ class Scanner { // Literal buffer support inline void StartLiteral() { - LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? - &literal_buffer2_ : &literal_buffer1_; + LiteralBuffer* free_buffer = + (current_.literal_chars == &literal_buffer0_) + ? &literal_buffer1_ + : (current_.literal_chars == &literal_buffer1_) ? &literal_buffer2_ + : &literal_buffer0_; free_buffer->Reset(); next_.literal_chars = free_buffer; } inline void StartRawLiteral() { LiteralBuffer* free_buffer = - (current_.raw_literal_chars == &raw_literal_buffer1_) ? - &raw_literal_buffer2_ : &raw_literal_buffer1_; + (current_.raw_literal_chars == &raw_literal_buffer0_) + ? &raw_literal_buffer1_ + : (current_.raw_literal_chars == &raw_literal_buffer1_) + ? &raw_literal_buffer2_ + : &raw_literal_buffer0_; free_buffer->Reset(); next_.raw_literal_chars = free_buffer; } @@ -725,6 +696,7 @@ class Scanner { UnicodeCache* unicode_cache_; // Buffers collecting literal strings, numbers, etc. + LiteralBuffer literal_buffer0_; LiteralBuffer literal_buffer1_; LiteralBuffer literal_buffer2_; @@ -733,11 +705,13 @@ class Scanner { LiteralBuffer source_mapping_url_; // Buffer to store raw string values + LiteralBuffer raw_literal_buffer0_; LiteralBuffer raw_literal_buffer1_; LiteralBuffer raw_literal_buffer2_; - TokenDesc current_; // desc for current token (as returned by Next()) - TokenDesc next_; // desc for next token (one token look-ahead) + TokenDesc current_; // desc for current token (as returned by Next()) + TokenDesc next_; // desc for next token (one token look-ahead) + TokenDesc next_next_; // desc for the token after next (after PeakAhead()) // Variables for Scanner::BookmarkScope and the *Bookmark implementation. // These variables contain the scanner state when a bookmark is set. |