typescriptdomain.py (19435B)
1 """ 2 TypeScript domain. 3 4 :copyright: Copyright 2019 by Taler Systems SA 5 :license: LGPLv3+ 6 :author: Florian Dold 7 """ 8 9 import re 10 11 from pathlib import Path 12 13 from docutils import nodes 14 from typing import List, Optional, Iterable, Dict, Tuple 15 from typing import cast 16 17 from pygments.lexers import get_lexer_by_name 18 from pygments.filter import Filter 19 from pygments.token import Literal, Text, Operator, Keyword, Name, Number 20 from pygments.token import Comment, Token, _TokenType 21 from pygments.token import * 22 from pygments.lexer import RegexLexer, bygroups, include 23 from pygments.formatters import HtmlFormatter 24 25 from docutils import nodes 26 from docutils.nodes import Element, Node 27 28 from sphinx.roles import XRefRole 29 from sphinx.domains import Domain, ObjType, Index 30 from sphinx.directives import directives 31 from sphinx.util.docutils import SphinxDirective 32 from sphinx.util.nodes import make_refnode 33 from sphinx.util import logging 34 from sphinx.highlighting import PygmentsBridge 35 from sphinx.builders.html import StandaloneHTMLBuilder 36 from sphinx.pygments_styles import SphinxStyle 37 38 logger = logging.getLogger(__name__) 39 40 41 class TypeScriptDefinition(SphinxDirective): 42 """ 43 Directive for a code block with special highlighting or line numbering 44 settings. 45 """ 46 47 has_content = True 48 required_arguments = 1 49 optional_arguments = 0 50 final_argument_whitespace = False 51 option_spec = { 52 "force": directives.flag, 53 "linenos": directives.flag, 54 "dedent": int, 55 "lineno-start": int, 56 "emphasize-lines": directives.unchanged_required, 57 "caption": directives.unchanged_required, 58 "class": directives.class_option, 59 } 60 61 def run(self) -> List[Node]: 62 document = self.state.document 63 code = "\n".join(self.content) 64 location = self.state_machine.get_source_and_line(self.lineno) 65 66 linespec = self.options.get("emphasize-lines") 67 if linespec: 68 try: 69 nlines = len(self.content) 70 hl_lines = parselinenos(linespec, nlines) 71 if any(i >= nlines for i in hl_lines): 72 logger.warning( 73 __("line number spec is out of range(1-%d): %r") 74 % (nlines, self.options["emphasize-lines"]), 75 location=location, 76 ) 77 78 hl_lines = [x + 1 for x in hl_lines if x < nlines] 79 except ValueError as err: 80 return [document.reporter.warning(err, line=self.lineno)] 81 else: 82 hl_lines = None 83 84 if "dedent" in self.options: 85 location = self.state_machine.get_source_and_line(self.lineno) 86 lines = code.split("\n") 87 lines = dedent_lines(lines, self.options["dedent"], location=location) 88 code = "\n".join(lines) 89 90 literal = nodes.literal_block(code, code) # type: Element 91 if "linenos" in self.options or "lineno-start" in self.options: 92 literal["linenos"] = True 93 literal["classes"] += self.options.get("class", []) 94 literal["force"] = "force" in self.options 95 literal["language"] = "tsref" 96 extra_args = literal["highlight_args"] = {} 97 if hl_lines is not None: 98 extra_args["hl_lines"] = hl_lines 99 if "lineno-start" in self.options: 100 extra_args["linenostart"] = self.options["lineno-start"] 101 self.set_source_info(literal) 102 103 caption = self.options.get("caption") 104 if caption: 105 try: 106 literal = container_wrapper(self, literal, caption) 107 except ValueError as exc: 108 return [document.reporter.warning(exc, line=self.lineno)] 109 110 tsid = "tsref-type-" + self.arguments[0] 111 literal["ids"].append(tsid) 112 113 tsname = self.arguments[0] 114 ts = self.env.get_domain("ts") 115 ts.add_object("type", tsname, self.env.docname, tsid) 116 117 return [literal] 118 119 120 class TypeScriptDomain(Domain): 121 """TypeScript domain.""" 122 123 name = "ts" 124 label = "TypeScript" 125 126 directives = { 127 "def": TypeScriptDefinition, 128 } 129 130 roles = { 131 "type": XRefRole( 132 lowercase=False, warn_dangling=True, innernodeclass=nodes.inline 133 ), 134 } 135 136 dangling_warnings = { 137 "type": "undefined TypeScript type: %(target)s", 138 } 139 140 def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): 141 try: 142 info = self.objects[(str(typ), str(target))] 143 except KeyError: 144 logger.warn("type {}/{} not found".format(typ, target)) 145 return None 146 else: 147 anchor = "tsref-type-{}".format(str(target)) 148 title = typ.upper() + " " + target 149 return make_refnode(builder, fromdocname, info[0], anchor, contnode, title) 150 151 def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode): 152 """Resolve the pending_xref *node* with the given *target*. 153 154 The reference comes from an "any" or similar role, which means that Sphinx 155 don't know the type. 156 157 For now sphinxcontrib-httpdomain doesn't resolve any xref nodes. 158 159 :return: 160 list of tuples ``('domain:role', newnode)``, where ``'domain:role'`` 161 is the name of a role that could have created the same reference, 162 """ 163 ret = [] 164 try: 165 info = self.objects[("type", str(target))] 166 except KeyError: 167 pass 168 else: 169 anchor = "tsref-type-{}".format(str(target)) 170 title = "TYPE" + " " + target 171 node = make_refnode(builder, fromdocname, info[0], anchor, contnode, title) 172 ret.append(("ts:type", node)) 173 return ret 174 175 @property 176 def objects(self) -> Dict[Tuple[str, str], Tuple[str, str]]: 177 return self.data.setdefault( 178 "objects", {} 179 ) # (objtype, name) -> docname, labelid 180 181 def add_object(self, objtype: str, name: str, docname: str, labelid: str) -> None: 182 self.objects[objtype, name] = (docname, labelid) 183 184 185 class BetterTypeScriptLexer(RegexLexer): 186 """ 187 For `TypeScript <https://www.typescriptlang.org/>`_ source code. 188 """ 189 190 name = "TypeScript" 191 aliases = ["ts"] 192 filenames = ["*.ts"] 193 mimetypes = ["text/x-typescript"] 194 195 flags = re.DOTALL 196 tokens = { 197 "commentsandwhitespace": [ 198 (r"\s+", Text), 199 (r"<!--", Comment), 200 (r"//.*?\n", Comment.Single), 201 (r"/\*.*?\*/", Comment.Multiline), 202 ], 203 "slashstartsregex": [ 204 include("commentsandwhitespace"), 205 ( 206 r"/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/" r"([gim]+\b|\B)", 207 String.Regex, 208 "#pop", 209 ), 210 (r"(?=/)", Text, ("#pop", "badregex")), 211 (r"", Text, "#pop"), 212 ], 213 "badregex": [(r"\n", Text, "#pop")], 214 "typeexp": [ 215 (r"[a-zA-Z0-9_?.$]+", Keyword.Type), 216 (r"\s+", Text), 217 (r"[|]", Text), 218 (r"\n", Text, "#pop"), 219 (r";", Text, "#pop"), 220 (r"", Text, "#pop"), 221 ], 222 "root": [ 223 (r"^(?=\s|/|<!--)", Text, "slashstartsregex"), 224 include("commentsandwhitespace"), 225 ( 226 r"\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|" 227 r"(<<|>>>?|==?|!=?|[-<>+*%&\|\^/])=?", 228 Operator, 229 "slashstartsregex", 230 ), 231 (r"[{(\[;,]", Punctuation, "slashstartsregex"), 232 (r"[})\].]", Punctuation), 233 ( 234 r"(for|in|while|do|break|return|continue|switch|case|default|if|else|" 235 r"throw|try|catch|finally|new|delete|typeof|instanceof|void|" 236 r"this)\b", 237 Keyword, 238 "slashstartsregex", 239 ), 240 ( 241 r"(var|let|const|with|function)\b", 242 Keyword.Declaration, 243 "slashstartsregex", 244 ), 245 ( 246 r"(abstract|boolean|byte|char|class|const|debugger|double|enum|export|" 247 r"extends|final|float|goto|implements|import|int|interface|long|native|" 248 r"package|private|protected|public|short|static|super|synchronized|throws|" 249 r"transient|volatile)\b", 250 Keyword.Reserved, 251 ), 252 (r"(true|false|null|NaN|Infinity|undefined)\b", Keyword.Constant), 253 ( 254 r"(Array|Boolean|Date|Error|Function|Math|netscape|" 255 r"Number|Object|Packages|RegExp|String|sun|decodeURI|" 256 r"decodeURIComponent|encodeURI|encodeURIComponent|" 257 r"Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|" 258 r"window)\b", 259 Name.Builtin, 260 ), 261 # Match stuff like: module name {...} 262 ( 263 r"\b(module)(\s*)(\s*[a-zA-Z0-9_?.$][\w?.$]*)(\s*)", 264 bygroups(Keyword.Reserved, Text, Name.Other, Text), 265 "slashstartsregex", 266 ), 267 # Match variable type keywords 268 (r"\b(string|bool|number)\b", Keyword.Type), 269 # Match stuff like: constructor 270 (r"\b(constructor|declare|interface|as|AS)\b", Keyword.Reserved), 271 # Match stuff like: super(argument, list) 272 ( 273 r"(super)(\s*)\(([a-zA-Z0-9,_?.$\s]+\s*)\)", 274 bygroups(Keyword.Reserved, Text), 275 "slashstartsregex", 276 ), 277 # Match stuff like: function() {...} 278 (r"([a-zA-Z_?.$][\w?.$]*)\(\) \{", Name.Other, "slashstartsregex"), 279 # Match stuff like: (function: return type) 280 ( 281 r"([a-zA-Z0-9_?.$][\w?.$]*)(\s*:\s*)", 282 bygroups(Name.Other, Text), 283 "typeexp", 284 ), 285 # Match stuff like: type Foo = Bar | Baz 286 ( 287 r"\b(type)(\s*)([a-zA-Z0-9_?.$]+)(\s*)(=)(\s*)", 288 bygroups(Keyword.Reserved, Text, Name.Other, Text, Operator, Text), 289 "typeexp", 290 ), 291 (r"[$a-zA-Z_][a-zA-Z0-9_]*", Name.Other), 292 (r"[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?", Number.Float), 293 (r"0x[0-9a-fA-F]+", Number.Hex), 294 (r"[0-9]+", Number.Integer), 295 (r'"(\\\\|\\"|[^"])*"', String.Double), 296 (r"'(\\\\|\\'|[^'])*'", String.Single), 297 ], 298 } 299 300 301 # Map from token id to props. 302 # Properties can't be added to tokens 303 # since they derive from Python's tuple. 304 token_props = {} 305 306 307 class LinkFilter(Filter): 308 def __init__(self, app, **options): 309 self.app = app 310 Filter.__init__(self, **options) 311 312 def _filter_one_literal(self, ttype, value): 313 last = 0 314 for m in re.finditer(literal_reg, value): 315 pre = value[last : m.start()] 316 if pre: 317 yield ttype, pre 318 t = copy_token(ttype) 319 tok_setprop(t, "is_literal", True) 320 yield t, m.group(1) 321 last = m.end() 322 post = value[last:] 323 if post: 324 yield ttype, post 325 326 def filter(self, lexer, stream): 327 for ttype, value in stream: 328 if ttype in Token.Keyword.Type: 329 t = copy_token(ttype) 330 tok_setprop(t, "xref", value.strip()) 331 tok_setprop(t, "is_identifier", True) 332 yield t, value 333 elif ttype in Token.Comment: 334 last = 0 335 for m in re.finditer(link_reg, value): 336 pre = value[last : m.start()] 337 if pre: 338 yield from self._filter_one_literal(ttype, pre) 339 t = copy_token(ttype) 340 x1, x2 = m.groups() 341 x0 = m.group(0) 342 if x2 is None: 343 caption = x1.strip() 344 xref = x1.strip() 345 else: 346 caption = x1.strip() 347 xref = x2.strip() 348 tok_setprop(t, "xref", xref) 349 tok_setprop(t, "caption", caption) 350 if x0.endswith("_"): 351 tok_setprop(t, "trailing_underscore", True) 352 yield t, m.group(1) 353 last = m.end() 354 post = value[last:] 355 if post: 356 yield from self._filter_one_literal(ttype, post) 357 else: 358 yield ttype, value 359 360 361 _escape_html_table = { 362 ord("&"): u"&", 363 ord("<"): u"<", 364 ord(">"): u">", 365 ord('"'): u""", 366 ord("'"): u"'", 367 } 368 369 370 class LinkingHtmlFormatter(HtmlFormatter): 371 def __init__(self, **kwargs): 372 super(LinkingHtmlFormatter, self).__init__(**kwargs) 373 self._builder = kwargs["_builder"] 374 self._bridge = kwargs["_bridge"] 375 376 def _get_value(self, value, tok): 377 xref = tok_getprop(tok, "xref") 378 caption = tok_getprop(tok, "caption") 379 380 if tok_getprop(tok, "is_literal"): 381 return '<span style="font-weight: bolder">%s</span>' % (value,) 382 383 if tok_getprop(tok, "trailing_underscore"): 384 logger.warn( 385 "{}:{}: code block contains xref to '{}' with unsupported trailing underscore".format( 386 self._bridge.path, self._bridge.line, xref 387 ) 388 ) 389 390 if tok_getprop(tok, "is_identifier"): 391 if xref.startswith('"'): 392 return value 393 if re.match("^[0-9]+$", xref) is not None: 394 return value 395 if xref in ( 396 "number", 397 "object", 398 "string", 399 "boolean", 400 "any", 401 "true", 402 "false", 403 "null", 404 "undefined", 405 "Array", 406 "unknown", 407 ): 408 return value 409 410 if self._bridge.docname is None: 411 return value 412 if xref is None: 413 return value 414 content = caption if caption is not None else value 415 ts = self._builder.env.get_domain("ts") 416 r1 = ts.objects.get(("type", xref), None) 417 if r1 is not None: 418 rel_uri = ( 419 self._builder.get_relative_uri(self._bridge.docname, r1[0]) 420 + "#" 421 + r1[1] 422 ) 423 return ( 424 '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>' 425 % (rel_uri, content) 426 ) 427 428 std = self._builder.env.get_domain("std") 429 r2 = std.labels.get(xref.lower(), None) 430 if r2 is not None: 431 rel_uri = ( 432 self._builder.get_relative_uri(self._bridge.docname, r2[0]) 433 + "#" 434 + r2[1] 435 ) 436 return ( 437 '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>' 438 % (rel_uri, content) 439 ) 440 r3 = std.anonlabels.get(xref.lower(), None) 441 if r3 is not None: 442 rel_uri = ( 443 self._builder.get_relative_uri(self._bridge.docname, r3[0]) 444 + "#" 445 + r3[1] 446 ) 447 return ( 448 '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>' 449 % (rel_uri, content) 450 ) 451 452 logger.warn( 453 "{}:{}: code block contains unresolved xref '{}'".format( 454 self._bridge.path, self._bridge.line, xref 455 ) 456 ) 457 458 return value 459 460 def _fmt(self, value, tok): 461 cls = self._get_css_class(tok) 462 value = self._get_value(value, tok) 463 if cls is None or cls == "": 464 return value 465 return '<span class="%s">%s</span>' % (cls, value) 466 467 def _format_lines(self, tokensource): 468 """ 469 Just format the tokens, without any wrapping tags. 470 Yield individual lines. 471 """ 472 lsep = self.lineseparator 473 escape_table = _escape_html_table 474 475 line = "" 476 for ttype, value in tokensource: 477 link = get_annotation(ttype, "link") 478 479 parts = value.translate(escape_table).split("\n") 480 481 if len(parts) == 0: 482 # empty token, usually should not happen 483 pass 484 elif len(parts) == 1: 485 # no newline before or after token 486 line += self._fmt(parts[0], ttype) 487 else: 488 line += self._fmt(parts[0], ttype) 489 yield 1, line + lsep 490 for part in parts[1:-1]: 491 yield 1, self._fmt(part, ttype) + lsep 492 line = self._fmt(parts[-1], ttype) 493 494 if line: 495 yield 1, line + lsep 496 497 498 class MyPygmentsBridge(PygmentsBridge): 499 def __init__(self, builder, trim_doctest_flags): 500 self.dest = "html" 501 self.trim_doctest_flags = trim_doctest_flags 502 self.formatter_args = { 503 "style": SphinxStyle, 504 "_builder": builder, 505 "_bridge": self, 506 } 507 self.formatter = LinkingHtmlFormatter 508 self.builder = builder 509 self.path = None 510 self.line = None 511 self.docname = None 512 513 def highlight_block( 514 self, source, lang, opts=None, force=False, location=None, **kwargs 515 ): 516 if isinstance(location, tuple): 517 docname, line = location 518 self.line = line 519 self.path = self.builder.env.doc2path(docname) 520 self.docname = docname 521 elif isinstance(location, Element): 522 self.line = location.line 523 self.path = location.source 524 self.docname = self.builder.env.path2doc(self.path) 525 return super().highlight_block(source, lang, opts, force, location, **kwargs) 526 527 528 class MyHtmlBuilder(StandaloneHTMLBuilder): 529 name = "html-linked" 530 531 def init_highlighter(self): 532 if self.config.pygments_style is not None: 533 style = self.config.pygments_style 534 elif self.theme: 535 style = self.theme.get_confstr("theme", "pygments_style", "none") 536 else: 537 style = "sphinx" 538 self.highlighter = MyPygmentsBridge(self, self.config.trim_doctest_flags) 539 self.dark_highlighter = None 540 541 542 def get_annotation(tok, key): 543 if not hasattr(tok, "kv"): 544 return None 545 return tok.kv.get(key) 546 547 548 def copy_token(tok): 549 new_tok = _TokenType(tok) 550 # This part is very fragile against API changes ... 551 new_tok.subtypes = set(tok.subtypes) 552 new_tok.parent = tok.parent 553 return new_tok 554 555 556 def tok_setprop(tok, key, value): 557 tokid = id(tok) 558 e = token_props.get(tokid) 559 if e is None: 560 e = token_props[tokid] = (tok, {}) 561 _, kv = e 562 kv[key] = value 563 564 565 def tok_getprop(tok, key): 566 tokid = id(tok) 567 e = token_props.get(tokid) 568 if e is None: 569 return None 570 _, kv = e 571 return kv.get(key) 572 573 574 link_reg = re.compile(r"(?<!`)`([^`<]+)\s*(?:<([^>]+)>)?\s*`_?") 575 literal_reg = re.compile(r"``([^`]+)``") 576 577 578 def setup(app): 579 580 class TsrefLexer(BetterTypeScriptLexer): 581 def __init__(self, **options): 582 super().__init__(**options) 583 self.add_filter(LinkFilter(app)) 584 585 app.add_lexer("tsref", TsrefLexer) 586 app.add_domain(TypeScriptDomain) 587 app.add_builder(MyHtmlBuilder)