typescriptdomain.py (23930B)
1 """ 2 TypeScript domain. 3 4 :copyright: Copyright 2019 by Taler Systems SA 5 :license: LGPLv3+ 6 :author: Florian Dold 7 """ 8 9 import re 10 11 from docutils import nodes 12 from typing import Dict, Iterator, List, Tuple 13 14 from pygments.filter import Filter 15 from pygments.token import ( 16 Comment, 17 Keyword, 18 Name, 19 Number, 20 Operator, 21 Punctuation, 22 String, 23 Text, 24 Token, 25 _TokenType, 26 ) 27 from pygments.lexer import RegexLexer, bygroups, include 28 from pygments.formatters import HtmlFormatter 29 30 from docutils.nodes import Element, Node 31 32 from sphinx.roles import XRefRole 33 from sphinx.domains import Domain, ObjType 34 from sphinx.directives import directives 35 from sphinx.directives.code import ( 36 container_wrapper, 37 dedent_lines, 38 parse_line_num_spec, 39 ) 40 from sphinx.locale import __ 41 from sphinx.util.docutils import SphinxDirective 42 from sphinx.util.nodes import make_refnode 43 from sphinx.util import logging 44 from sphinx.highlighting import PygmentsBridge 45 46 logger = logging.getLogger(__name__) 47 48 49 class TypeScriptDefinition(SphinxDirective): 50 """ 51 Directive for a code block with special highlighting or line numbering 52 settings. 53 """ 54 55 has_content = True 56 required_arguments = 1 57 optional_arguments = 0 58 final_argument_whitespace = False 59 option_spec = { 60 "force": directives.flag, 61 "linenos": directives.flag, 62 "dedent": int, 63 "lineno-start": int, 64 "emphasize-lines": directives.unchanged_required, 65 "caption": directives.unchanged_required, 66 "class": directives.class_option, 67 } 68 69 def run(self) -> List[Node]: 70 document = self.state.document 71 code = "\n".join(self.content) 72 location = self.state_machine.get_source_and_line(self.lineno) 73 74 linespec = self.options.get("emphasize-lines") 75 if linespec: 76 try: 77 nlines = len(self.content) 78 hl_lines = parse_line_num_spec(linespec, nlines) 79 if any(i >= nlines for i in hl_lines): 80 logger.warning( 81 __("line number spec is out of range(1-%d): %r"), 82 nlines, 83 self.options["emphasize-lines"], 84 location=location, 85 ) 86 87 hl_lines = [x + 1 for x in hl_lines if x < nlines] 88 except ValueError as err: 89 return [document.reporter.warning(err, line=self.lineno)] 90 else: 91 hl_lines = None 92 93 if "dedent" in self.options: 94 location = self.state_machine.get_source_and_line(self.lineno) 95 lines = code.splitlines(True) 96 lines = dedent_lines(lines, self.options["dedent"], location=location) 97 code = "".join(lines) 98 99 literal = nodes.literal_block(code, code) # type: Element 100 if "linenos" in self.options or "lineno-start" in self.options: 101 literal["linenos"] = True 102 literal["classes"] += self.options.get("class", []) 103 literal["force"] = "force" in self.options 104 literal["language"] = "tsref" 105 extra_args = literal["highlight_args"] = {} 106 if hl_lines is not None: 107 extra_args["hl_lines"] = hl_lines 108 if "lineno-start" in self.options: 109 extra_args["linenostart"] = self.options["lineno-start"] 110 self.set_source_info(literal) 111 112 caption = self.options.get("caption") 113 if caption: 114 try: 115 literal = container_wrapper(self, literal, caption) 116 except ValueError as exc: 117 return [document.reporter.warning(exc, line=self.lineno)] 118 119 tsid = "tsref-type-" + self.arguments[0] 120 literal["ids"].append(tsid) 121 122 tsname = self.arguments[0] 123 ts = self.env.get_domain("ts") 124 ts.add_object("type", tsname, self.env.docname, tsid) 125 126 return [literal] 127 128 129 class TypeScriptDomain(Domain): 130 """TypeScript domain.""" 131 132 name = "ts" 133 label = "TypeScript" 134 object_types = { 135 "type": ObjType("type", "type"), 136 } 137 initial_data = { 138 "objects": {}, 139 } 140 141 directives = { 142 "def": TypeScriptDefinition, 143 } 144 145 roles = { 146 "type": XRefRole( 147 lowercase=False, warn_dangling=True, innernodeclass=nodes.inline 148 ), 149 } 150 151 dangling_warnings = { 152 "type": "undefined TypeScript type: %(target)s", 153 } 154 155 def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): 156 info = self.find_object(str(typ), str(target), fromdocname) 157 if info is None: 158 return None 159 title = typ.upper() + " " + target 160 return make_refnode(builder, fromdocname, info[0], info[1], contnode, title) 161 162 def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode): 163 """Resolve the pending_xref *node* with the given *target*. 164 165 The reference comes from an "any" or similar role, which means that Sphinx 166 don't know the type. 167 168 For now sphinxcontrib-httpdomain doesn't resolve any xref nodes. 169 170 :return: 171 list of tuples ``('domain:role', newnode)``, where ``'domain:role'`` 172 is the name of a role that could have created the same reference, 173 """ 174 ret = [] 175 info = self.find_object("type", str(target), fromdocname) 176 if info is not None: 177 title = "TYPE" + " " + target 178 node = make_refnode(builder, fromdocname, info[0], info[1], contnode, title) 179 ret.append(("ts:type", node)) 180 return ret 181 182 @property 183 def objects(self) -> Dict[Tuple[str, str], List[Tuple[str, str]]]: 184 """Map ``(object type, name)`` to all documents defining it.""" 185 186 objects = self.data.setdefault("objects", {}) 187 # Environments written by the old extension stored just one tuple. 188 for key, value in list(objects.items()): 189 if isinstance(value, tuple): 190 objects[key] = [value] 191 return objects 192 193 def add_object(self, objtype: str, name: str, docname: str, labelid: str) -> None: 194 locations = self.objects.setdefault((objtype, name), []) 195 location = (docname, labelid) 196 if location not in locations: 197 locations.append(location) 198 199 def find_object( 200 self, objtype: str, name: str, fromdocname: str 201 ) -> Tuple[str, str] | None: 202 locations = self.objects.get((objtype, name), []) 203 for location in locations: 204 if location[0] == fromdocname: 205 return location 206 if locations: 207 return sorted(locations)[0] 208 return None 209 210 def clear_doc(self, docname: str) -> None: 211 for key, locations in list(self.objects.items()): 212 remaining = [location for location in locations if location[0] != docname] 213 if remaining: 214 self.objects[key] = remaining 215 else: 216 del self.objects[key] 217 218 def merge_domaindata(self, docnames, otherdata) -> None: 219 for (objtype, name), locations in otherdata.get("objects", {}).items(): 220 if isinstance(locations, tuple): 221 locations = [locations] 222 for docname, labelid in locations: 223 if docname in docnames: 224 self.add_object(objtype, name, docname, labelid) 225 226 def get_objects(self) -> Iterator[Tuple[str, str, str, str, str, int]]: 227 for (objtype, name), locations in self.objects.items(): 228 for docname, labelid in locations: 229 yield name, name, objtype, docname, labelid, 1 230 231 232 class BetterTypeScriptLexer(RegexLexer): 233 """ 234 For `TypeScript <https://www.typescriptlang.org/>`_ source code. 235 """ 236 237 name = "TypeScript" 238 aliases = ["ts"] 239 filenames = ["*.ts"] 240 mimetypes = ["text/x-typescript"] 241 242 flags = re.DOTALL 243 tokens = { 244 "commentsandwhitespace": [ 245 (r"\s+", Text), 246 (r"<!--", Comment), 247 (r"//.*?\n", Comment.Single), 248 (r"/\*.*?\*/", Comment.Multiline), 249 ], 250 "slashstartsregex": [ 251 include("commentsandwhitespace"), 252 ( 253 r"/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/" r"([gim]+\b|\B)", 254 String.Regex, 255 "#pop", 256 ), 257 (r"(?=/)", Text, ("#pop", "badregex")), 258 (r"", Text, "#pop"), 259 ], 260 "badregex": [(r"\n", Text, "#pop")], 261 "typeexp": [ 262 include("commentsandwhitespace"), 263 (r"`(?:\\.|[^`])*`", String.Backtick), 264 (r'"(\\\\|\\"|[^"])*"', String.Double), 265 (r"'(\\\\|\\'|[^'])*'", String.Single), 266 (r";", Punctuation, "#pop"), 267 # Object-property names occur inside inline type literals. Leave 268 # those as ordinary names; their value type remains in this state. 269 (r"[$a-zA-Z_][a-zA-Z0-9_$]*(?=\s*\??\s*:)", Name.Other), 270 (r"[$a-zA-Z_][a-zA-Z0-9_$]*(?:\.[$a-zA-Z_][a-zA-Z0-9_$]*)*", Keyword.Type), 271 (r"[{}()\[\],.?]", Punctuation), 272 (r"[|&<>=:+*\-/]", Operator), 273 (r"[0-9]+", Number.Integer), 274 (r".", Text), 275 ], 276 "heritage": [ 277 include("commentsandwhitespace"), 278 (r"{", Punctuation, "#pop"), 279 (r"[$a-zA-Z_][a-zA-Z0-9_$]*(?:\.[$a-zA-Z_][a-zA-Z0-9_$]*)*", Keyword.Type), 280 (r"[<>,.?\[\]&|]", Punctuation), 281 (r".", Text), 282 ], 283 "root": [ 284 (r"^(?=\s|/|<!--)", Text, "slashstartsregex"), 285 include("commentsandwhitespace"), 286 # TypeScript template literal types and string templates. Full 287 # interpolation highlighting is unnecessary here, but recognizing 288 # the complete literal avoids falling back to relaxed lexing. 289 (r"`(?:\\.|[^`])*`", String.Backtick), 290 # A reserved word can still be an object-property name (for 291 # example ``class``). Enter the type state when its colon arrives. 292 (r"(:)(\s*)", bygroups(Text, Text), "typeexp"), 293 # Resume a multi-line union/intersection after an inline object 294 # member caused the type state to end at its semicolon. 295 (r"([|&])(\s*)", bygroups(Operator, Text), "typeexp"), 296 ( 297 r"\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|" 298 r"(<<|>>>?|==?|!=?|[-<>+*%&\|\^/])=?", 299 Operator, 300 "slashstartsregex", 301 ), 302 (r"[{(\[;,]", Punctuation, "slashstartsregex"), 303 (r"[})\].]", Punctuation), 304 ( 305 r"(for|in|while|do|break|return|continue|switch|case|default|if|else|" 306 r"throw|try|catch|finally|new|delete|typeof|instanceof|void|" 307 r"this)\b", 308 Keyword, 309 "slashstartsregex", 310 ), 311 ( 312 r"(var|let|const|with|function)\b", 313 Keyword.Declaration, 314 "slashstartsregex", 315 ), 316 ( 317 r"(abstract|boolean|byte|char|class|const|debugger|double|enum|export|" 318 r"final|float|goto|import|int|interface|long|native|" 319 r"package|private|protected|public|short|static|super|synchronized|throws|" 320 r"transient|volatile)\b", 321 Keyword.Reserved, 322 ), 323 (r"(true|false|null|NaN|Infinity|undefined)\b", Keyword.Constant), 324 ( 325 r"(Array|Boolean|Date|Error|Function|Math|netscape|" 326 r"Number|Object|Packages|RegExp|String|sun|decodeURI|" 327 r"decodeURIComponent|encodeURI|encodeURIComponent|" 328 r"Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|" 329 r"window)\b", 330 Name.Builtin, 331 ), 332 # Match stuff like: module name {...} 333 ( 334 r"\b(module)(\s*)(\s*[a-zA-Z0-9_?.$][\w?.$]*)(\s*)", 335 bygroups(Keyword.Reserved, Text, Name.Other, Text), 336 "slashstartsregex", 337 ), 338 # Match variable type keywords 339 (r"\b(string|bool|number)\b", Keyword.Type), 340 # Match stuff like: constructor 341 (r"\b(constructor|declare|interface|as|AS)\b", Keyword.Reserved), 342 # Match interface/class heritage clauses. 343 ( 344 r"\b(extends|implements)(\s+)", 345 bygroups(Keyword.Reserved, Text), 346 "heritage", 347 ), 348 # Match stuff like: super(argument, list) 349 ( 350 r"(super)(\s*)\(([a-zA-Z0-9,_?.$\s]+\s*)\)", 351 bygroups(Keyword.Reserved, Text), 352 "slashstartsregex", 353 ), 354 # Match stuff like: function() {...} 355 (r"([a-zA-Z_?.$][\w?.$]*)\(\) \{", Name.Other, "slashstartsregex"), 356 # Match stuff like: (function: return type) 357 ( 358 r"([a-zA-Z0-9_?.$][\w?.$]*)(\s*:\s*)", 359 bygroups(Name.Other, Text), 360 "typeexp", 361 ), 362 # Match stuff like: type Foo = Bar | Baz 363 ( 364 r"\b(type)(\s+)([$a-zA-Z_][a-zA-Z0-9_$]*)([^=]*)(=)(\s*)", 365 bygroups(Keyword.Reserved, Text, Name.Other, Text, Operator, Text), 366 "typeexp", 367 ), 368 (r"[$a-zA-Z_][a-zA-Z0-9_]*", Name.Other), 369 (r"[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?", Number.Float), 370 (r"0x[0-9a-fA-F]+", Number.Hex), 371 (r"[0-9]+", Number.Integer), 372 (r'"(\\\\|\\"|[^"])*"', String.Double), 373 (r"'(\\\\|\\'|[^'])*'", String.Single), 374 ], 375 } 376 377 378 # Map from token id to props. 379 # Properties can't be added to tokens 380 # since they derive from Python's tuple. 381 token_props = {} 382 383 384 class LinkFilter(Filter): 385 def _filter_one_literal(self, ttype, value): 386 last = 0 387 for m in re.finditer(literal_reg, value): 388 pre = value[last : m.start()] 389 if pre: 390 yield ttype, pre 391 t = copy_token(ttype) 392 tok_setprop(t, "is_literal", True) 393 yield t, m.group(1) 394 last = m.end() 395 post = value[last:] 396 if post: 397 yield ttype, post 398 399 def filter(self, lexer, stream): 400 for ttype, value in stream: 401 if ttype in Token.Keyword.Type: 402 t = copy_token(ttype) 403 tok_setprop(t, "xref", value.strip()) 404 tok_setprop(t, "is_identifier", True) 405 tok_setprop(t, "optional_xref", True) 406 yield t, value 407 elif ttype in Token.Comment: 408 last = 0 409 for m in re.finditer(link_reg, value): 410 pre = value[last : m.start()] 411 if pre: 412 yield from self._filter_one_literal(ttype, pre) 413 t = copy_token(ttype) 414 x1, x2 = m.groups() 415 x0 = m.group(0) 416 if x2 is None: 417 caption = x1.strip() 418 xref = x1.strip() 419 else: 420 caption = x1.strip() 421 xref = x2.strip() 422 tok_setprop(t, "xref", xref) 423 tok_setprop(t, "caption", caption) 424 if x0.endswith("_"): 425 tok_setprop(t, "trailing_underscore", True) 426 elif x2 is None: 427 # A bare single-backtick span is also how Markdown/JSDoc 428 # writes inline code. Link it when a target exists, but 429 # only diagnose explicit reStructuredText references. 430 tok_setprop(t, "optional_xref", True) 431 yield t, m.group(1) 432 last = m.end() 433 post = value[last:] 434 if post: 435 yield from self._filter_one_literal(ttype, post) 436 else: 437 yield ttype, value 438 439 440 _escape_html_table = { 441 ord("&"): "&", 442 ord("<"): "<", 443 ord(">"): ">", 444 ord('"'): """, 445 ord("'"): "'", 446 } 447 448 449 class LinkingHtmlFormatter(HtmlFormatter): 450 def __init__(self, **kwargs): 451 super(LinkingHtmlFormatter, self).__init__(**kwargs) 452 self._builder = kwargs["_builder"] 453 self._bridge = kwargs["_bridge"] 454 455 def _get_value(self, value, tok): 456 xref = tok_getprop(tok, "xref") 457 caption = tok_getprop(tok, "caption") 458 459 if tok_getprop(tok, "is_literal"): 460 return '<span style="font-weight: bolder">%s</span>' % (value,) 461 462 if tok_getprop(tok, "trailing_underscore"): 463 logger.warning( 464 "{}:{}: code block contains xref to '{}' with unsupported trailing underscore".format( 465 self._bridge.path, self._bridge.line, xref 466 ) 467 ) 468 469 if tok_getprop(tok, "is_identifier"): 470 if not xref or xref.startswith('"'): 471 return value 472 if re.match("^[0-9]+$", xref) is not None: 473 return value 474 475 if self._bridge.docname is None: 476 return value 477 if xref is None: 478 return value 479 content = caption if caption is not None else value 480 ts = self._builder.env.get_domain("ts") 481 r1 = ts.find_object("type", xref, self._bridge.docname) 482 # Qualified type references are currently one lexer token. Link 483 # ``Namespace.Member`` to the closest documented prefix if the member 484 # itself is not registered as a standalone declaration. 485 if r1 is None and tok_getprop(tok, "is_identifier") and "." in xref: 486 parts = xref.split(".") 487 for end in range(len(parts) - 1, 0, -1): 488 r1 = ts.find_object("type", ".".join(parts[:end]), self._bridge.docname) 489 if r1 is not None: 490 break 491 if r1 is not None: 492 rel_uri = ( 493 self._builder.get_relative_uri(self._bridge.docname, r1[0]) 494 + "#" 495 + r1[1] 496 ) 497 return ( 498 '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>' 499 % (rel_uri, content) 500 ) 501 502 if tok_getprop(tok, "is_identifier") and tok_getprop(tok, "optional_xref"): 503 return value 504 505 std = self._builder.env.get_domain("std") 506 r2 = std.labels.get(xref.lower(), None) 507 if r2 is not None: 508 rel_uri = ( 509 self._builder.get_relative_uri(self._bridge.docname, r2[0]) 510 + "#" 511 + r2[1] 512 ) 513 return ( 514 '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>' 515 % (rel_uri, content) 516 ) 517 r3 = std.anonlabels.get(xref.lower(), None) 518 if r3 is not None: 519 rel_uri = ( 520 self._builder.get_relative_uri(self._bridge.docname, r3[0]) 521 + "#" 522 + r3[1] 523 ) 524 return ( 525 '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>' 526 % (rel_uri, content) 527 ) 528 529 if not tok_getprop(tok, "optional_xref"): 530 logger.warning( 531 "{}:{}: code block contains unresolved xref '{}'".format( 532 self._bridge.path, self._bridge.line, xref 533 ) 534 ) 535 536 return value 537 538 def _fmt(self, value, tok): 539 cls = self._get_css_class(tok) 540 value = self._get_value(value, tok) 541 if cls is None or cls == "": 542 return value 543 return '<span class="%s">%s</span>' % (cls, value) 544 545 def _format_lines(self, tokensource): 546 """ 547 Just format the tokens, without any wrapping tags. 548 Yield individual lines. 549 """ 550 lsep = self.lineseparator 551 escape_table = _escape_html_table 552 553 line = "" 554 for ttype, value in tokensource: 555 parts = value.translate(escape_table).split("\n") 556 557 if len(parts) == 0: 558 # empty token, usually should not happen 559 pass 560 elif len(parts) == 1: 561 # no newline before or after token 562 line += self._fmt(parts[0], ttype) 563 else: 564 line += self._fmt(parts[0], ttype) 565 yield 1, line + lsep 566 for part in parts[1:-1]: 567 yield 1, self._fmt(part, ttype) + lsep 568 line = self._fmt(parts[-1], ttype) 569 570 if line: 571 yield 1, line + lsep 572 573 574 class LinkingPygmentsBridge(PygmentsBridge): 575 def __init__(self, builder, style): 576 self.dest = "html" 577 self.latex_engine = None 578 self.formatter_args = { 579 "style": style, 580 "_builder": builder, 581 "_bridge": self, 582 } 583 self.formatter = LinkingHtmlFormatter 584 self.builder = builder 585 self.path = None 586 self.line = None 587 self.docname = None 588 589 def highlight_block( 590 self, source, lang, opts=None, force=False, location=None, **kwargs 591 ): 592 self.path = None 593 self.line = None 594 self.docname = None 595 if isinstance(location, tuple): 596 docname, line = location 597 self.line = line 598 self.path = self.builder.env.doc2path(docname) 599 self.docname = docname 600 elif isinstance(location, Element): 601 self.line = location.line 602 self.path = location.source 603 self.docname = self.builder.env.path2doc(self.path) 604 # The path/line above point at the source file the code block was 605 # written in (used for diagnostics). However, relative links must be 606 # computed against the document that is *currently being written* -- 607 # not the (possibly ``.. include``-d) source file, which may live in a 608 # deeper directory and would inject spurious "../" segments into every 609 # cross reference. Prefer the builder's current output docname. 610 current = getattr(self.builder, "current_docname", None) 611 if current: 612 self.docname = current 613 return super().highlight_block(source, lang, opts, force, location, **kwargs) 614 615 616 def install_linking_highlighters(app): 617 """Wrap the highlighters created by any standard HTML-family builder.""" 618 619 builder = app.builder 620 if builder.format != "html": 621 return 622 623 def replace(highlighter): 624 if highlighter is None: 625 return None 626 style = highlighter.formatter_args["style"] 627 return LinkingPygmentsBridge(builder, style) 628 629 builder.highlighter = replace(builder.highlighter) 630 builder.dark_highlighter = replace(getattr(builder, "dark_highlighter", None)) 631 632 633 def copy_token(tok): 634 new_tok = _TokenType(tok) 635 # This part is very fragile against API changes ... 636 new_tok.subtypes = set(tok.subtypes) 637 new_tok.parent = tok.parent 638 return new_tok 639 640 641 def tok_setprop(tok, key, value): 642 tokid = id(tok) 643 e = token_props.get(tokid) 644 if e is None: 645 e = token_props[tokid] = (tok, {}) 646 _, kv = e 647 kv[key] = value 648 649 650 def tok_getprop(tok, key): 651 tokid = id(tok) 652 e = token_props.get(tokid) 653 if e is None: 654 return None 655 _, kv = e 656 return kv.get(key) 657 658 659 link_reg = re.compile(r"(?<!`)`([^`<]+)\s*(?:<([^>]+)>)?\s*`_?") 660 literal_reg = re.compile(r"``([^`]+)``") 661 662 663 def setup(app): 664 665 class TsrefLexer(BetterTypeScriptLexer): 666 def __init__(self, **options): 667 super().__init__(**options) 668 self.add_filter(LinkFilter()) 669 670 app.add_lexer("tsref", TsrefLexer) 671 app.add_domain(TypeScriptDomain) 672 app.connect("builder-inited", install_linking_highlighters) 673 return { 674 "parallel_read_safe": True, 675 "parallel_write_safe": True, 676 }