commit 7391444394e0b69101d1cc83d6af0437701a9b78
parent e12f1ea1358a2330717c58784647ba43c5855e8a
Author: Nullptrderef <nullptrderef@proton.me>
Date: Sat, 29 Jun 2024 18:28:08 +0200
fix: type extractor "works"
Diffstat:
3 files changed, 54 insertions(+), 277 deletions(-)
diff --git a/contrib/type-extractor/.gitignore b/contrib/type-extractor/.gitignore
@@ -1 +1,2 @@
-node_modules
-\ No newline at end of file
+node_modules
+dist
diff --git a/contrib/type-extractor/dist/main.mjs b/contrib/type-extractor/dist/main.mjs
@@ -1,216 +0,0 @@
-// src/main.ts
-import fsSync, { promises as fs } from "fs";
-import ts from "typescript";
-import * as path from "path";
-var ignoredExports = ["PublishedAgeRestrictionBaseKey"];
-var runFileJob = (file) => {
- let workingFile = file;
- const tsDefs = file.match(/[\t ]*\.\. ts\:def\:\: [a-zA-Z][a-zA-Z0-9_]*/g);
- const defines = [];
- let dtsOutput = "";
- if (tsDefs)
- for (const def of tsDefs) {
- if (!def) {
- console.warn("No matches in ", file);
- break;
- }
- workingFile = workingFile.substring(workingFile.indexOf(def));
- let [defMatch, indentation, defName] = def.match(/([\t ])*\.\. ts\:def\:\: ([a-zA-Z][a-zA-Z0-9_]*) *\n?/) ?? [];
- if (!defMatch || !indentation || !defName || ignoredExports.includes(defName))
- continue;
- indentation = indentation ?? "";
- workingFile = workingFile.substring(defMatch.length);
- const workingFileLines = workingFile.split("\n");
- let tsMatch = "";
- while (workingFileLines[0]?.trim() === "") workingFileLines.shift();
- while ((workingFileLines[0]?.trim() === "" || workingFileLines[0] && new RegExp("^" + "[ \\t]".repeat(indentation.length + 2)).test(
- workingFileLines[0]
- )) && !workingFileLines[0]?.trim()?.startsWith(".. ts:def::")) {
- if (workingFileLines[0].length > indentation.length + 2)
- workingFileLines[0] = workingFileLines[0].substring(
- indentation.length + 2
- );
- tsMatch += workingFileLines.shift() + "\n";
- }
- workingFile = workingFileLines.join("\n");
- tsMatch = tsMatch.replace(/([ \t]*\/\/.*\n?)+/g, (match) => {
- match = match.split("\n").map((v) => v.replace(/[ \t]+\/\/ ?/, "").trim()).join("\n").trim();
- if (match.includes("\n"))
- match = `/**
-${match.split("\n").map((v) => v.trimStart().startsWith("//") ? v.replace("//", "") : v).map((v) => " *" + (v.startsWith(" ") ? "" : " ") + v).join("\n").replace(/\*\//g, "*\u200B/")}
- */
-`;
- else
- match = `/**
- * ${(match.trimStart().startsWith("//") ? match.replace("//", "") : match).trim().replace(/\*\//g, "*\u200B/")}
- */
-`;
- return match;
- }).trim();
- defines.push(defName);
- dtsOutput += tsMatch + "\n";
- }
- if (defines.length === 0) {
- console.warn(file, new Error("No Defines"));
- return null;
- }
- dtsOutput += `
-export { ${defines.join(", ")} };
-`;
- return {
- defines,
- dtsOutput
- };
-};
-(async () => {
- const genDocsForDirs = ["core/"].map((v) => path.resolve(process.argv[2], v));
- const genDocsForFiles = (await Promise.all(
- genDocsForDirs.map(
- async (dir) => (await fs.readdir(dir)).map((file) => path.join(dir, file))
- )
- )).flat();
- const output = path.resolve(
- process.env.TYPE_OUTPUT ?? process.env.TMP ?? process.env.TEMP ?? "/tmp",
- "net.taler.docs.ts-extracted"
- );
- const tsDocOutput = path.join(output, "dts");
- if (fsSync.existsSync(tsDocOutput))
- await fs.rm(tsDocOutput, { recursive: true });
- await fs.mkdir(tsDocOutput, {
- recursive: true
- });
- const jobResults = (await Promise.all(
- genDocsForFiles.map(async (filepath) => ({
- source: filepath,
- output: path.join(
- tsDocOutput,
- path.basename(filepath).replace(".rst", ".ts")
- ),
- result: runFileJob(await fs.readFile(filepath, "utf-8"))
- }))
- )).filter((v) => v.result !== null);
- jobResults.push({
- source: "/tmp/net.taler.docs.extracted/_forced_polyfill",
- output: path.join(tsDocOutput, "post-polyfill.ts"),
- // This polyfill overwrites any object defined elsewhere
- result: runFileJob(`
-.. ts:def:: Integer
- // An integer value.
- // @integer
- type Integer = number;
-`)
- });
- jobResults.unshift({
- source: "/tmp/net.taler.docs.extracted/_polyfill",
- output: path.join(tsDocOutput, "polyfill.ts"),
- // This polyfill can be overwritten by the actual docs; it's contents will be outputted but ignored by the import resolver if overwritten
- result: runFileJob(`
-.. ts:def:: PaytoHash
- // A Binary Object
- type PaytoHash = string;
-.. ts:def:: AgeCommitmentHash
- // A Binary Object
- type AgeCommitmentHash = string;
-.. ts:def:: TALER_RefreshCommitmentP
- // A Binary Object
- type TALER_RefreshCommitmentP = string;
-.. ts:def:: WireTransferIdentifierRawP
- // A Binary Object
- type WireTransferIdentifierRawP = string;
-.. ts:def:: Base32
- // Binary data is generally encoded using Crockford\u2019s variant of Base32 (https://www.crockford.com/wrmg/base32.html), except that \u201CU\u201D is not excluded but also decodes to \u201CV\u201D to make OCR easy. We will still simply use the JSON type \u201Cbase32\u201D and the term \u201CCrockford Base32\u201D in the text to refer to the resulting encoding.
- type Base32 = string;
-.. ts:def:: ExtensionManifest
- // Mostly undocumented object; see {@link https://docs.taler.net/design-documents/006-extensions.html#extensionmanifest-object} for what it likely is?
- interface ExtensionManifest {
- // The criticality of the extension MUST be provided. It has the same
- // semantics as "critical" has for extensions in X.509:
- // - if "true", the client must "understand" the extension before
- // proceeding,
- // - if "false", clients can safely skip extensions they do not
- // understand.
- // (see https://datatracker.ietf.org/doc/html/rfc5280#section-4.2)
- critical: boolean;
-
- // The version information MUST be provided in Taler's protocol version
- // ranges notation, see
- // https://docs.taler.net/core/api-common.html#protocol-version-ranges
- version: LibtoolVersion;
-
- // Optional configuration object, defined by the feature itself
- config?: object;
- }
-.. ts:def:: WadId
- // https://docs.taler.net/core/api-common.html#wadid
- type WadId = string;
-.. ts:def:: ContractChoice
- // Untyped in documentation https://docs.taler.net/design-documents/046-mumimo-contracts.html#tsref-type-ContractChoice
- type ContractChoice = any;
-`)
- });
- const fileByExport = {};
- jobResults.forEach((result) => {
- result.result.defines.forEach(
- (define) => fileByExport[define] = result.output
- );
- });
- await Promise.all(
- jobResults.map((result) => {
- const src = result.result.dtsOutput;
- const toBeImported = [];
- const sourceFile = ts.createSourceFile(
- path.basename(result.output),
- src,
- {
- languageVersion: ScriptTarget.ESNext
- }
- );
- const astWalker = (node) => {
- if (node.kind === ts.SyntaxKind.TypeReference) {
- const typeRefNode = node;
- const { typeName } = typeRefNode;
- const identifier = "escapedText" in typeName ? typeName.escapedText : typeName.getText();
- if (!result.result.defines.includes(`${identifier}`))
- toBeImported.push(`${identifier}`);
- }
- ts.forEachChild(node, astWalker);
- };
- astWalker(sourceFile);
- result.result.dtsOutput = `${toBeImported.filter((v, i, a) => a.indexOf(v) === i).map((v) => {
- if (fileByExport[v])
- return `import { ${v} } from ${JSON.stringify(
- "./" + path.basename(fileByExport[v])
- )}`;
- else if (["String", "Boolean"].includes(v))
- console.warn(
- `In file ${result.source}: Please use ${v.toLocaleLowerCase()} instead of ${v}`
- );
- console.warn("Could not find reference to", v);
- return "// WARN: UNKNOWN REF: " + JSON.stringify(v);
- }).join("\n")}
-${result.result.dtsOutput}`;
- })
- );
- await Promise.all(
- jobResults.map(async ({ output: output2, result }) => {
- await fs.writeFile(output2, result.dtsOutput);
- })
- );
- const exportsByFile = {};
- for (const [exported, file] of Object.entries(fileByExport)) {
- exportsByFile[file] = exportsByFile[file] ?? [];
- exportsByFile[file].push(exported);
- }
- await fs.writeFile(
- path.join(tsDocOutput, "main.ts"),
- Object.entries(exportsByFile).map(
- ([file, exports]) => (
- // We could use export * from, but then we'd get class conflicts if 2 separate files declare the same type - including if our polyfill overwrites or gets overwritten
- `export { ${exports.join(", ")} } from ${JSON.stringify(
- "./" + path.basename(file)
- // TODO: use path.relative
- )};`
- )
- ).join("")
- );
-})();
diff --git a/contrib/type-extractor/src/main.ts b/contrib/type-extractor/src/main.ts
@@ -1,7 +1,7 @@
// Usage: $0 <path to documentation root>
import fsSync, { promises as fs } from 'fs';
-import ts, { type ScriptTarget } from 'typescript';
+import ts from 'typescript';
import * as path from 'path';
const ignoredExports = ['PublishedAgeRestrictionBaseKey'];
@@ -15,57 +15,50 @@ const runFileJob = (file: string) => {
const tsDefs = file.match(/[\t ]*\.\. ts\:def\:\: [a-zA-Z][a-zA-Z0-9_]*/g);
const defines: string[] = [];
let dtsOutput = '';
- if (tsDefs)
- for (const def of tsDefs) {
- if (!def) {
- console.warn('No matches in ', file);
- break;
- }
- workingFile = workingFile.substring(workingFile.indexOf(def));
- let [defMatch, indentation, defName] =
- def.match(/([\t ])*\.\. ts\:def\:\: ([a-zA-Z][a-zA-Z0-9_]*) *\n?/) ??
- [];
+ if (!tsDefs) return null;
+ for (const def of tsDefs) {
+ if (!def) {
+ console.warn('No matches in ', file);
+ break;
+ }
+ workingFile = workingFile.substring(workingFile.indexOf(def));
+ let [defMatch, indentation, defName] =
+ def.match(/([\t ])*\.\. ts\:def\:\: ([a-zA-Z][a-zA-Z0-9_]*) *\n?/) ?? [];
- if (
- !defMatch ||
- !indentation ||
- !defName ||
- ignoredExports.includes(defName)
- )
- continue;
+ if (!defMatch || !defName || ignoredExports.includes(defName)) continue;
- // Extract the ts def
- indentation = indentation ?? '';
- workingFile = workingFile.substring(defMatch.length);
- const workingFileLines = workingFile.split('\n');
- let tsMatch = '';
- while (workingFileLines[0]?.trim() === '') workingFileLines.shift();
- while (
- (workingFileLines[0]?.trim() === '' ||
- (workingFileLines[0] &&
- new RegExp('^' + '[ \\t]'.repeat(indentation.length + 2)).test(
- workingFileLines[0],
- ))) &&
- !workingFileLines[0]?.trim()?.startsWith('.. ts:def::')
- ) {
- if (workingFileLines[0].length > indentation.length + 2)
- workingFileLines[0] = workingFileLines[0].substring(
- indentation.length + 2,
- );
- tsMatch += workingFileLines.shift() + '\n';
- }
- workingFile = workingFileLines.join('\n');
+ // Extract the ts def
+ indentation = indentation ?? '';
+ workingFile = workingFile.substring(defMatch.length);
+ const workingFileLines = workingFile.split('\n');
+ let tsMatch = '';
+ while (workingFileLines[0]?.trim() === '') workingFileLines.shift();
+ while (
+ (workingFileLines[0]?.trim() === '' ||
+ (workingFileLines[0] &&
+ new RegExp('^' + '[ \\t]'.repeat(indentation.length + 2)).test(
+ workingFileLines[0],
+ ))) &&
+ !workingFileLines[0]?.trim()?.startsWith('.. ts:def::')
+ ) {
+ if (workingFileLines[0].length > indentation.length + 2)
+ workingFileLines[0] = workingFileLines[0].substring(
+ indentation.length + 2,
+ );
+ tsMatch += workingFileLines.shift() + '\n';
+ }
+ workingFile = workingFileLines.join('\n');
- // Convert comments to JSDocs
- tsMatch = tsMatch
- .replace(/([ \t]*\/\/.*\n?)+/g, (match) => {
- match = match
- .split('\n')
- .map((v) => v.replace(/[ \t]+\/\/ ?/, '').trim())
- .join('\n')
- .trim();
- if (match.includes('\n'))
- match = `/**
+ // Convert comments to JSDocs
+ tsMatch = tsMatch
+ .replace(/([ \t]*\/\/.*\n?)+/g, (match) => {
+ match = match
+ .split('\n')
+ .map((v) => v.replace(/[ \t]+\/\/ ?/, '').trim())
+ .join('\n')
+ .trim();
+ if (match.includes('\n'))
+ match = `/**
${match
.split('\n')
.map((v) => (v.trimStart().startsWith('//') ? v.replace('//', '') : v))
@@ -74,20 +67,20 @@ ${match
.replace(/\*\//g, '*/')}
*/
`;
- else
- match = `/**
+ else
+ match = `/**
* ${(match.trimStart().startsWith('//') ? match.replace('//', '') : match)
.trim()
.replace(/\*\//g, '*/')}
*/
`;
- return match;
- })
- .trim();
+ return match;
+ })
+ .trim();
- defines.push(defName);
- dtsOutput += tsMatch + '\n';
- }
+ defines.push(defName);
+ dtsOutput += tsMatch + '\n';
+ }
if (defines.length === 0) return null; // nothing to give back, just exit
@@ -140,7 +133,7 @@ export { ${defines.join(', ')} };
result: runFileJob(await fs.readFile(filepath, 'utf-8'))!,
})),
)
- ).filter((v) => v.result !== null);
+ ).filter((v) => v.result !== null && v.result !== undefined);
// Polyfilling!!!
// TODO: Extract these to standalone .rst files!
jobResults.push({
@@ -221,7 +214,7 @@ export { ${defines.join(', ')} };
path.basename(result.output),
src,
{
- languageVersion: ScriptTarget.ESNext,
+ languageVersion: ts.ScriptTarget.ESNext,
},
);
