tools: Include links to source code in documentation

Parse source code using acorn; extracting exports.  When producing
documentation, match exports to headers.  When a match is found, add a [src]
link.

This first commit handles simple exported classes and functions, and does so
without requiring any changes to the source code or markdown.  Subsequent
commits will attempt to match more headers, and some of these changes are
likely to require changes to the source code and/or markdown.

PR-URL: https://github.com/nodejs/node/pull/22405
Reviewed-By: James M Snell <jasnell@gmail.com>
Reviewed-By: Refael Ackermann <refack@gmail.com>
Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com>

13 files changed, 252 insertions, 4 deletions

diff --git a/Makefile b/Makefile
index 2452244fc2..8769b5b95c 100644
--- a/Makefile
+++ b/Makefile
@@ -645,10 +645,15 @@ out/doc/api/assets/%: doc/api_assets/% out/doc/api/assets
 run-npm-ci = $(PWD)/$(NPM) ci
 
 gen-api = tools/doc/generate.js --node-version=$(FULLVERSION) \
+		--apilinks=out/apilinks.json \
 		--analytics=$(DOCS_ANALYTICS) $< --output-directory=out/doc/api
+gen-apilink = tools/doc/apilinks.js $(wildcard lib/*.js) > $@
+
+out/apilinks.json: $(wildcard lib/*.js) tools/doc/apilinks.js
+	$(call available-node, $(gen-apilink))
 
 out/doc/api/%.json out/doc/api/%.html: doc/api/%.md tools/doc/generate.js \
-	tools/doc/html.js tools/doc/json.js
+	tools/doc/html.js tools/doc/json.js | out/apilinks.json
 	$(call available-node, $(gen-api))
 
 out/doc/api/all.html: $(apidocs_html) tools/doc/allhtml.js
diff --git a/doc/api_assets/style.css b/doc/api_assets/style.css
index f59f377004..7d65b7405b 100644
--- a/doc/api_assets/style.css
+++ b/doc/api_assets/style.css
@@ -283,6 +283,11 @@ h2, h3, h4, h5 {
   padding-right: 40px;
 }
 
+.srclink {
+  float: right;
+  font-size: smaller;
+}
+
 h1 span, h2 span, h3 span, h4 span {
   position: absolute;
   display: block;
diff --git a/test/doctool/test-apilinks.js b/test/doctool/test-apilinks.js
new file mode 100644
index 0000000000..e53c81a08a
--- /dev/null
+++ b/test/doctool/test-apilinks.js
@@ -0,0 +1,39 @@
+'use strict';
+
+require('../common');
+const fixtures = require('../common/fixtures');
+const fs = require('fs');
+const assert = require('assert');
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const script = path.join(__dirname, '..', '..', 'tools', 'doc', 'apilinks.js');
+
+const apilinks = fixtures.path('apilinks');
+fs.readdirSync(apilinks).forEach((fixture) => {
+  if (!fixture.endsWith('.js')) return;
+  const file = path.join(apilinks, fixture);
+
+  const expectedContent = fs.readFileSync(file + 'on', 'utf8');
+
+  const output = execFileSync(
+    process.execPath,
+    [script, file],
+    { encoding: 'utf-8' }
+  );
+
+  const expectedLinks = JSON.parse(expectedContent);
+  const actualLinks = JSON.parse(output);
+
+  for (const [k, v] of Object.entries(expectedLinks)) {
+    assert.ok(k in actualLinks, `link not found: ${k}`);
+    assert.ok(actualLinks[k].endsWith('/' + v),
+              `link ${actualLinks[k]} expected to end with ${v}`);
+    delete actualLinks[k];
+  }
+
+  assert.strictEqual(
+    Object.keys(actualLinks).length, 0,
+    `unexpected links returned ${JSON.stringify(actualLinks)}`
+  );
+});
diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js
index 8c05ea6a0b..2fcb8315af 100644
--- a/test/doctool/test-doctool-html.js
+++ b/test/doctool/test-doctool-html.js
@@ -28,7 +28,7 @@ function toHTML({ input, filename, nodeVersion, analytics }, cb) {
     .use(html.firstHeader)
     .use(html.preprocessText)
     .use(html.preprocessElements, { filename })
-    .use(html.buildToc, { filename })
+    .use(html.buildToc, { filename, apilinks: {} })
     .use(remark2rehype, { allowDangerousHTML: true })
     .use(raw)
     .use(htmlStringify)
diff --git a/test/fixtures/apilinks/buffer.js b/test/fixtures/apilinks/buffer.js
new file mode 100644
index 0000000000..8ee44123a5
--- /dev/null
+++ b/test/fixtures/apilinks/buffer.js
@@ -0,0 +1,12 @@
+'use strict';
+
+// Buffer instance methods are exported as 'buf'.
+
+function Buffer() {
+}
+
+Buffer.prototype.instanceMethod = function() {}
+
+module.exports = {
+  Buffer
+};
diff --git a/test/fixtures/apilinks/buffer.json b/test/fixtures/apilinks/buffer.json
new file mode 100644
index 0000000000..528eca6eb2
--- /dev/null
+++ b/test/fixtures/apilinks/buffer.json
@@ -0,0 +1,4 @@
+{
+  "buffer.Buffer": "buffer.js#L5",
+  "buf.instanceMethod": "buffer.js#L8"
+}
diff --git a/test/fixtures/apilinks/mod.js b/test/fixtures/apilinks/mod.js
new file mode 100644
index 0000000000..72606121f5
--- /dev/null
+++ b/test/fixtures/apilinks/mod.js
@@ -0,0 +1,11 @@
+'use strict';
+
+// A module may export one or more methods.
+
+function foo() {
+}
+
+
+module.exports = {
+  foo
+};
diff --git a/test/fixtures/apilinks/mod.json b/test/fixtures/apilinks/mod.json
new file mode 100644
index 0000000000..7d803e6236
--- /dev/null
+++ b/test/fixtures/apilinks/mod.json
@@ -0,0 +1,3 @@
+{
+  "mod.foo": "mod.js#L5"
+}
diff --git a/test/fixtures/apilinks/prototype.js b/test/fixtures/apilinks/prototype.js
new file mode 100644
index 0000000000..40218e844e
--- /dev/null
+++ b/test/fixtures/apilinks/prototype.js
@@ -0,0 +1,13 @@
+'use strict';
+
+// An exported class using classic prototype syntax.
+
+function Class() {
+}
+
+Class.classMethod = function() {}
+Class.prototype.instanceMethod = function() {}
+
+module.exports = {
+  Class
+};
diff --git a/test/fixtures/apilinks/prototype.json b/test/fixtures/apilinks/prototype.json
new file mode 100644
index 0000000000..d61c32da62
--- /dev/null
+++ b/test/fixtures/apilinks/prototype.json
@@ -0,0 +1,5 @@
+{
+  "prototype.Class": "prototype.js#L5",
+  "Class.classMethod": "prototype.js#L8",
+  "class.instanceMethod": "prototype.js#L9"
+}
diff --git a/tools/doc/apilinks.js b/tools/doc/apilinks.js
new file mode 100644
index 0000000000..98dd7827d6
--- /dev/null
+++ b/tools/doc/apilinks.js
@@ -0,0 +1,141 @@
+'use strict';
+
+// Scan API sources for definitions.
+//
+// Note the output is produced based on a world class parser, adherence to
+// conventions, and a bit of guess work. Examples:
+//
+//  * We scan for top level module.exports statements, and determine what
+//    is exported by looking at the source code only (i.e., we don't do
+//    an eval). If exports include `Foo`, it probably is a class, whereas
+//    if what is exported is `constants` it probably is prefixed by the
+//    basename of the source file (e.g., `zlib`), unless that source file is
+//    `buffer.js`, in which case the name is just `buf`.  unless the constant
+//    is `kMaxLength`, in which case it is `buffer`.
+//
+//  * We scan for top level definitions for those exports, handling
+//    most common cases (e.g., `X.prototype.foo =`, `X.foo =`,
+//    `function X(...) {...}`). Over time, we expect to handle more
+//    cases (example: ES2015 class definitions).
+
+const acorn = require('../../deps/acorn');
+const fs = require('fs');
+const path = require('path');
+const child_process = require('child_process');
+
+// Run a command, capturing stdout, ignoring errors.
+function execSync(command) {
+  try {
+    return child_process.execSync(
+      command,
+      { stdio: ['ignore', null, 'ignore'] }
+    ).toString().trim();
+  } catch {
+    return '';
+  }
+}
+
+// Determine orign repo and tag (or hash) of the most recent commit.
+const local_branch = execSync('git name-rev --name-only HEAD');
+const tracking_remote = execSync(`git config branch.${local_branch}.remote`);
+const remote_url = execSync(`git config remote.${tracking_remote}.url`);
+const repo = (remote_url.match(/(\w+\/\w+)\.git\r?\n?$/) ||
+             ['', 'nodejs/node'])[1];
+
+const hash = execSync('git log -1 --pretty=%H') || 'master';
+const tag = execSync(`git describe --contains ${hash}`).split('\n')[0] || hash;
+
+// Extract definitions from each file specified.
+const definition = {};
+process.argv.slice(2).forEach((file) => {
+  const basename = path.basename(file, '.js');
+
+  // Parse source.
+  const source = fs.readFileSync(file, 'utf8');
+  const ast = acorn.parse(source, { ecmaVersion: 10, locations: true });
+  const program = ast.body;
+
+  // Scan for exports.
+  const exported = { constructors: [], identifiers: [] };
+  program.forEach((statement) => {
+    if (statement.type !== 'ExpressionStatement') return;
+    const expr = statement.expression;
+    if (expr.type !== 'AssignmentExpression') return;
+
+    let lhs = expr.left;
+    if (expr.left.object.type === 'MemberExpression') lhs = lhs.object;
+    if (lhs.type !== 'MemberExpression') return;
+    if (lhs.object.name !== 'module') return;
+    if (lhs.property.name !== 'exports') return;
+
+    let rhs = expr.right;
+    while (rhs.type === 'AssignmentExpression') rhs = rhs.right;
+
+    if (rhs.type === 'NewExpression') {
+      exported.constructors.push(rhs.callee.name);
+    } else if (rhs.type === 'ObjectExpression') {
+      rhs.properties.forEach((property) => {
+        if (property.value.type === 'Identifier') {
+          exported.identifiers.push(property.value.name);
+          if (/^[A-Z]/.test(property.value.name[0])) {
+            exported.constructors.push(property.value.name);
+          }
+        }
+      });
+    } else if (rhs.type === 'Identifier') {
+      exported.identifiers.push(rhs.name);
+    }
+  });
+
+  // Scan for definitions matching those exports; currently supports:
+  //
+  //   ClassName.foo = ...;
+  //   ClassName.prototype.foo = ...;
+  //   function Identifier(...) {...};
+  //
+  const link = `https://github.com/${repo}/blob/${tag}/` +
+    path.relative('.', file).replace(/\\/g, '/');
+
+  program.forEach((statement) => {
+    if (statement.type === 'ExpressionStatement') {
+      const expr = statement.expression;
+      if (expr.type !== 'AssignmentExpression') return;
+      if (expr.left.type !== 'MemberExpression') return;
+
+      let object;
+      if (expr.left.object.type === 'MemberExpression') {
+        if (expr.left.object.property.name !== 'prototype') return;
+        object = expr.left.object.object;
+      } else if (expr.left.object.type === 'Identifier') {
+        object = expr.left.object;
+      } else {
+        return;
+      }
+
+      if (!exported.constructors.includes(object.name)) return;
+
+      let objectName = object.name;
+      if (expr.left.object.type === 'MemberExpression') {
+        objectName = objectName.toLowerCase();
+        if (objectName === 'buffer') objectName = 'buf';
+      }
+
+      let name = expr.left.property.name;
+      if (expr.left.computed) {
+        name = `${objectName}[${name}]`;
+      } else {
+        name = `${objectName}.${name}`;
+      }
+
+      definition[name] = `${link}#L${statement.loc.start.line}`;
+    } else if (statement.type === 'FunctionDeclaration') {
+      const name = statement.id.name;
+      if (!exported.identifiers.includes(name)) return;
+      if (basename.startsWith('_')) return;
+      definition[`${basename}.${name}`] =
+        `${link}#L${statement.loc.start.line}`;
+    }
+  });
+});
+
+console.log(JSON.stringify(definition, null, 2));
diff --git a/tools/doc/generate.js b/tools/doc/generate.js
index f2e3e8a164..28e09d003f 100644
--- a/tools/doc/generate.js
+++ b/tools/doc/generate.js
@@ -40,6 +40,7 @@ let filename = null;
 let nodeVersion = null;
 let analytics = null;
 let outputDir = null;
+let apilinks = {};
 
 args.forEach(function(arg) {
   if (!arg.startsWith('--')) {
@@ -50,6 +51,10 @@ args.forEach(function(arg) {
     analytics = arg.replace(/^--analytics=/, '');
   } else if (arg.startsWith('--output-directory=')) {
     outputDir = arg.replace(/^--output-directory=/, '');
+  } else if (arg.startsWith('--apilinks=')) {
+    apilinks = JSON.parse(
+      fs.readFileSync(arg.replace(/^--apilinks=/, ''), 'utf8')
+    );
   }
 });
 
@@ -71,7 +76,7 @@ fs.readFile(filename, 'utf8', (er, input) => {
     .use(json.jsonAPI, { filename })
     .use(html.firstHeader)
     .use(html.preprocessElements, { filename })
-    .use(html.buildToc, { filename })
+    .use(html.buildToc, { filename, apilinks })
     .use(remark2rehype, { allowDangerousHTML: true })
     .use(raw)
     .use(htmlStringify)
diff --git a/tools/doc/html.js b/tools/doc/html.js
index f1ac9e144e..899605437c 100644
--- a/tools/doc/html.js
+++ b/tools/doc/html.js
@@ -329,7 +329,7 @@ function versionSort(a, b) {
   return +b.match(numberRe)[0] - +a.match(numberRe)[0];
 }
 
-function buildToc({ filename }) {
+function buildToc({ filename, apilinks }) {
   return (tree, file) => {
     const startIncludeRefRE = /^\s*<!-- \[start-include:(.+)\] -->\s*$/;
     const endIncludeRefRE = /^\s*<!-- \[end-include:.+\] -->\s*$/;
@@ -376,6 +376,11 @@ function buildToc({ filename }) {
                   `id="${headingText}">#</a></span>`;
       }
 
+      const api = headingText.replace(/^.*:\s+/, '').replace(/\(.*/, '');
+      if (apilinks[api]) {
+        anchor = `<a class="srclink" href=${apilinks[api]}>[src]</a>${anchor}`;
+      }
+
       node.children.push({ type: 'html', value: anchor });
     });