tools: build all.json by combining generated JSON

Notes:

1) Removed a number of root properties that did not seem relevant:
   source, desc, and introduced_in.  There no longer is a source, and
   the other two are from the first include and do not reflect the
   entire API.

2) As with https://github.com/nodejs/node/issues/20100, the current
   "desc" properties sometimes contained in-page links, other times
   referenced another page, and often did not match the links in the
   original HTML or JSON file. I chose to standardize on external links
   as "desc" values are isolated snippets as opposed to all.html which
   can be viewed as a standalone and self contained document.

3) Eliminated preprocessing for @include entirely, including the test
   case for this function.

4) _toc.md was renamed to index.md.

5) index comments no longer appear in embedded TOCs (left hand side
   column in the generated documentation.

PR-URL: https://github.com/nodejs/node/pull/21637
Reviewed-By: Vse Mozhet Byt <vsemozhetbyt@gmail.com>
Reviewed-By: Rich Trott <rtrott@gmail.com>

3 files changed, 30 insertions, 46 deletions

diff --git a/test/doctool/test-doctool-html.js b/test/doctool/test-doctool-html.js
index 0ca46c0ade..a6119c1690 100644
--- a/test/doctool/test-doctool-html.js
+++ b/test/doctool/test-doctool-html.js
@@ -11,7 +11,6 @@ try {
 const assert = require('assert');
 const { readFile } = require('fs');
 const fixtures = require('../common/fixtures');
-const processIncludes = require('../../tools/doc/preprocess.js');
 const toHTML = require('../../tools/doc/html.js');
 
 // Test data is a list of objects with two properties.
@@ -65,15 +64,6 @@ const testData = [
       '<p>Describe <code>Something</code> in more detail here. </p>'
   },
   {
-    file: fixtures.path('doc_with_includes.md'),
-    html: '<!-- [start-include:doc_inc_1.md] -->' +
-    '<p>Look <a href="doc_inc_2.html#doc_inc_2_foobar">here</a>!</p>' +
-    '<!-- [end-include:doc_inc_1.md] --><!-- [start-include:doc_inc_2.md] -->' +
-    '<h1>foobar<span><a class="mark" href="#doc_inc_2_foobar" ' +
-    'id="doc_inc_2_foobar">#</a></span></h1>' +
-    '<p>I exist and am being linked to.</p><!-- [end-include:doc_inc_2.md] -->'
-  },
-  {
     file: fixtures.path('sample_document.md'),
     html: '<ol><li>fish</li><li><p>fish</p></li><li><p>Redfish</p></li>' +
       '<li>Bluefish</li></ol>',
@@ -90,36 +80,34 @@ testData.forEach(({ file, html, analyticsId }) => {
 
   readFile(file, 'utf8', common.mustCall((err, input) => {
     assert.ifError(err);
-    processIncludes(file, input, common.mustCall((err, preprocessed) => {
-      assert.ifError(err);
 
-      toHTML(
-        {
-          input: preprocessed,
-          filename: 'foo',
-          nodeVersion: process.version,
-          analytics: analyticsId,
-        },
-        common.mustCall((err, output) => {
-          assert.ifError(err);
+    toHTML(
+      {
+        input: input,
+        filename: 'foo',
+        nodeVersion: process.version,
+        analytics: analyticsId,
+      },
+      common.mustCall((err, output) => {
+        assert.ifError(err);
 
-          const actual = output.replace(spaces, '');
-          // Assert that the input stripped of all whitespace contains the
-          // expected markup.
-          assert(actual.includes(expected));
+        const actual = output.replace(spaces, '');
+        // Assert that the input stripped of all whitespace contains the
+        // expected markup.
+        assert(actual.includes(expected));
 
-          // Testing the insertion of Google Analytics script when
-          // an analytics id is provided. Should not be present by default.
-          const scriptDomain = 'google-analytics.com';
-          if (includeAnalytics) {
-            assert(actual.includes(scriptDomain),
-                   `Google Analytics script was not present in "${actual}"`);
-          } else {
-            assert.strictEqual(actual.includes(scriptDomain), false,
-                               'Google Analytics script was present in ' +
-                               `"${actual}"`);
-          }
-        }));
-    }));
+        // Testing the insertion of Google Analytics script when
+        // an analytics id is provided. Should not be present by default.
+        const scriptDomain = 'google-analytics.com';
+        if (includeAnalytics) {
+          assert(actual.includes(scriptDomain),
+                 `Google Analytics script was not present in "${actual}"`);
+        } else {
+          assert.strictEqual(actual.includes(scriptDomain), false,
+                             'Google Analytics script was present in ' +
+                             `"${actual}"`);
+        }
+      })
+    );
   }));
 });
diff --git a/test/doctool/test-make-doc.js b/test/doctool/test-make-doc.js
index e019288f75..a11da0e97d 100644
--- a/test/doctool/test-make-doc.js
+++ b/test/doctool/test-make-doc.js
@@ -13,17 +13,16 @@ const path = require('path');
 
 const apiPath = path.resolve(__dirname, '..', '..', 'out', 'doc', 'api');
 const allDocs = fs.readdirSync(apiPath);
-assert.ok(allDocs.includes('_toc.html'));
+assert.ok(allDocs.includes('index.html'));
 
 const actualDocs = allDocs.filter(
   (name) => {
     const extension = path.extname(name);
-    return (extension === '.html' || extension === '.json') &&
-           name !== '_toc.html';
+    return extension === '.html' || extension === '.json';
   }
 );
 
-const toc = fs.readFileSync(path.resolve(apiPath, '_toc.html'), 'utf8');
+const toc = fs.readFileSync(path.resolve(apiPath, 'index.html'), 'utf8');
 const re = /href="([^/]+\.html)"/;
 const globalRe = new RegExp(re, 'g');
 const links = toc.match(globalRe);
@@ -32,8 +31,7 @@ assert.notStrictEqual(links, null);
 // Filter out duplicate links, leave just filenames, add expected JSON files.
 const linkedHtmls = [...new Set(links)].map((link) => link.match(re)[1]);
 const expectedJsons = linkedHtmls
-                       .map((name) => name.replace('.html', '.json'))
-                       .concat('_toc.json');
+                       .map((name) => name.replace('.html', '.json'));
 const expectedDocs = linkedHtmls.concat(expectedJsons);
 
 // Test that all the relative links in the TOC match to the actual documents.
diff --git a/test/fixtures/doc_with_includes.md b/test/fixtures/doc_with_includes.md
deleted file mode 100644
index 901bf0f1b0..0000000000
--- a/test/fixtures/doc_with_includes.md
+++ /dev/null
@@ -1,2 +0,0 @@
-@include doc_inc_1
-@include doc_inc_2.md