diff options
Diffstat (limited to 'deps/v8/third_party/jinja2/lexer.py')
| -rw-r--r-- | deps/v8/third_party/jinja2/lexer.py | 57 |
1 files changed, 31 insertions, 26 deletions
diff --git a/deps/v8/third_party/jinja2/lexer.py b/deps/v8/third_party/jinja2/lexer.py index c8dac214ed..6fd135dd5b 100644 --- a/deps/v8/third_party/jinja2/lexer.py +++ b/deps/v8/third_party/jinja2/lexer.py @@ -11,18 +11,16 @@ operators we don't allow in templates. On the other hand it separates template code and python code in expressions. - :copyright: (c) 2010 by the Jinja Team. + :copyright: (c) 2017 by the Jinja Team. :license: BSD, see LICENSE for more details. """ import re - -from operator import itemgetter from collections import deque +from operator import itemgetter + +from jinja2._compat import implements_iterator, intern, iteritems, text_type from jinja2.exceptions import TemplateSyntaxError from jinja2.utils import LRUCache -from jinja2._compat import iteritems, implements_iterator, text_type, \ - intern, PY2 - # cache for the lexers. Exists in order to be able to have multiple # environments with the same lexer @@ -34,16 +32,25 @@ string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) integer_re = re.compile(r'\d+') -# we use the unicode identifier rule if this python version is able -# to handle unicode identifiers, otherwise the standard ASCII one. try: + # check if this Python supports Unicode identifiers compile('föö', '<unknown>', 'eval') except SyntaxError: - name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b') + # no Unicode support, use ASCII identifiers + name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*') + check_ident = False else: - from jinja2 import _stringdefs - name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start, - _stringdefs.xid_continue)) + # Unicode support, build a pattern to match valid characters, and set flag + # to use str.isidentifier to validate during lexing + from jinja2 import _identifier + name_re = re.compile(r'[\w{0}]+'.format(_identifier.pattern)) + check_ident = True + # remove the pattern from memory after building the regex + import sys + del sys.modules['jinja2._identifier'] + import jinja2 + del jinja2._identifier + del _identifier float_re = re.compile(r'(?<!\.)\d+\.\d+') newline_re = re.compile(r'(\r\n|\r|\n)') @@ -288,7 +295,7 @@ class TokenStreamIterator(object): @implements_iterator class TokenStream(object): - """A token stream is an iterable that yields :class:`Token`\s. The + """A token stream is an iterable that yields :class:`Token`\\s. The parser however does not iterate over it but calls :meth:`next` to go one token ahead. The current active token is stored as :attr:`current`. """ @@ -340,7 +347,10 @@ class TokenStream(object): return self.next_if(expr) is not None def __next__(self): - """Go one token ahead and return the old one""" + """Go one token ahead and return the old one. + + Use the built-in :func:`next` instead of calling this directly. + """ rv = self.current if self._pushed: self.current = self._pushed.popleft() @@ -500,7 +510,7 @@ class Lexer(object): ], # blocks TOKEN_BLOCK_BEGIN: [ - (c('(?:\-%s\s*|%s)%s' % ( + (c(r'(?:\-%s\s*|%s)%s' % ( e(environment.block_end_string), e(environment.block_end_string), block_suffix_re @@ -508,14 +518,14 @@ class Lexer(object): ] + tag_rules, # variables TOKEN_VARIABLE_BEGIN: [ - (c('\-%s\s*|%s' % ( + (c(r'\-%s\s*|%s' % ( e(environment.variable_end_string), e(environment.variable_end_string) )), TOKEN_VARIABLE_END, '#pop') ] + tag_rules, # raw block TOKEN_RAW_BEGIN: [ - (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( + (c(r'(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( e(environment.block_start_string), block_prefix_re, e(environment.block_end_string), @@ -565,6 +575,10 @@ class Lexer(object): token = value elif token == 'name': value = str(value) + if check_ident and not value.isidentifier(): + raise TemplateSyntaxError( + 'Invalid character in identifier', + lineno, name, filename) elif token == 'string': # try to unescape string try: @@ -574,15 +588,6 @@ class Lexer(object): except Exception as e: msg = str(e).split(':')[-1].strip() raise TemplateSyntaxError(msg, lineno, name, filename) - # if we can express it as bytestring (ascii only) - # we do that for support of semi broken APIs - # as datetime.datetime.strftime. On python 3 this - # call becomes a noop thanks to 2to3 - if PY2: - try: - value = value.encode('ascii') - except UnicodeError: - pass elif token == 'integer': value = int(value) elif token == 'float': |