commit aeea52790fa0d4a54f5839a1adce9eaace5a9dba parent b3a7295511a1e7ce7620e297038ae664b7ffe17c Author: Florian Dold <dold@taler.net> Date: Mon, 31 Aug 2026 12:56:14 +0200 test runner: summarize package results and remove stale outputs Diffstat:
25 files changed, 808 insertions(+), 23 deletions(-)
diff --git a/.gitignore b/.gitignore @@ -9,6 +9,8 @@ build/ test/ !packages/wallet-webui/test/ !packages/wallet-webui/test/** +!packages/qa-tooling/test/ +!packages/qa-tooling/test/** # GNU-style build system configure diff --git a/Makefile b/Makefile @@ -46,6 +46,7 @@ submodules-update: .PHONY: check check: pnpm install -r --frozen-lockfile + pnpm run clean pnpm run build pnpm run check diff --git a/README b/README @@ -139,6 +139,27 @@ Chrome users: - `Load unpacked` button in the upper left - Select `packages/wallet-webui/dist/extension/chrome/` +## Unit and package tests + +Run the repository check from a clean set of generated files with: + +```shell +make check +``` + +The command builds the workspace, runs every package test command, continues +after package failures, and prints one summary at the end. To rerun only one +package while working, use its workspace name, for example: + +```shell +pnpm --filter @gnu-taler/taler-wallet-core test +``` + +Package test commands are responsible for removing their own generated test +output, so direct package runs cannot pick up JavaScript left behind by an +older source tree. See the [test runner design](doc/testrunner.md) for the +output, isolation, and generated-file conventions. + # Integration Tests This repository comes with integration tests for GNU Taler. To run them, diff --git a/doc/testrunner.md b/doc/testrunner.md @@ -0,0 +1,121 @@ +# Test runner design + +## Goals + +The unit and package test workflow has three guarantees: + +1. A successful repository run is easy to scan: it shows package progress and + one summary instead of a TAP entry for every passing test. +2. All packages are attempted even when an earlier package fails, and the + final process status reports whether any package failed. +3. A test command runs only artifacts produced from the current source tree. + Deleted or renamed test sources must not survive as stale JavaScript and run + again. + +These guarantees apply to the Node-based unit and package tests in this +repository. The integration test runner described in the root README is a +separate workflow. + +## Commands + +The normal repository command is: + +```shell +make check +``` + +It installs the locked dependencies, cleans generated workspace files, builds +the workspace, and invokes the root package test orchestrator. The explicit +workspace clean is a defense in depth for all generated files, not the only +protection against stale tests. + +After a successful workspace build, rerun only the root test phase with: + +```shell +pnpm check +``` + +or run one package directly: + +```shell +pnpm --filter @gnu-taler/taler-util test +``` + +The root summary prints the direct package rerun command for every failure. + +## Why packages remain separate + +The root command does not put every JavaScript file into one Node test runner +process. Packages use different compilation and bundling configurations, have +different dependency graphs, and sometimes need package-local setup. Combining +all files would weaken those package boundaries and make the root runner aware +of implementation details that belong in each package. + +Instead, `packages/qa-tooling/bin/test-all.mjs` discovers workspace packages +with a `test` script, orders them after their tested workspace dependencies, +and executes those scripts sequentially. It does not stop at the first failed +package. A custom Node reporter suppresses successful per-test TAP output, +prints failure details immediately, and records counts that the orchestrator +combines into one final table. + +This is one repository-level execution and one consistent report, while each +package still owns its compiler, test glob, environment, and exit status. +Packages whose `test` script only checks compilation appear as `compile only` +because they do not produce Node test events. + +The concise reporter is supplied to child Node processes through +`NODE_OPTIONS`. A caller-provided `--test-reporter` option is rejected because +two reporters cannot reliably produce the repository summary. Run a package +directly when another Node reporter is needed for debugging. + +## Generated test output + +Every package test command must be hermetic with respect to its generated test +files. Before compiling or bundling tests, it removes the complete directory +that it owns and then recreates that directory from the current sources. + +The ownership conventions are: + +- TypeScript package tests that emit to `lib/` run `test:clean` first. That + removes `lib/` and the package TypeScript build-info file before invoking the + compiler. If a package's normal build produces additional module formats, + the test command recreates those formats before returning so dependents do + not see a partial package build. Coverage commands that compile tests follow + the same cleanup rule. +- Web UI test builds emit beneath `dist/test/`, never `dist/prod/`. The shared + `@gnu-taler/web-util/build` helper empties the configured test destination + before every build and refuses to remove the package root or a path outside + it. +- A package with a different generated-test directory must give that directory + a single clear owner and empty it at the start of every `test` command. +- Source files and hand-maintained fixtures must never be stored in a directory + owned by generated test output. + +Cleaning must happen inside the package test command, rather than relying only +on `make check`, because developers and CI jobs often rerun an individual +package. The root clean remains useful for catching unrelated stale build +artifacts before the production build. + +The QA tooling tests enforce the common TypeScript cleanup prefix and reject UI +test builders that target `dist/prod`. The shared UI cleanup helper also has +tests for deletion and path-safety behavior. + +## Adding or changing tests + +When adding a package test command: + +1. Put the complete build-and-run workflow in the package's `test` script. +2. Choose a generated directory owned only by that workflow and empty it before + compiling. Prefer `lib/` for the existing TypeScript emitter pattern and + `dist/test/` for UI bundles. +3. Use Node's test runner for the produced test files so the repository + reporter can collect consistent results. +4. Make the package command return nonzero for compilation failures, test + failures, and setup failures. +5. Verify both a direct package run and `pnpm check`. To exercise deletion, + rename or remove a test source and confirm its old generated file is absent + after the next direct package run. + +Keep informational output from successful tests small. Failure diagnostics can +be detailed; they are preserved by the concise reporter and followed by the +repository summary. diff --git a/package.json b/package.json @@ -8,8 +8,8 @@ "lint": "pnpm --filter @gnu-taler/qa-tooling install && ./packages/qa-tooling/bin/eslint.mjs .", "i18n:source2po": "pnpm run --filter '@gnu-taler/*' i18n:source2po", "i18n:po2strings": "pnpm run --filter '@gnu-taler/*' i18n:po2strings", - "check": "pnpm run --filter '@gnu-taler/*' --if-present --sequential --no-bail test", - "test": "pnpm run --filter '@gnu-taler/*' --if-present --sequential --no-bail test" + "check": "node ./packages/qa-tooling/bin/test-all.mjs", + "test": "node ./packages/qa-tooling/bin/test-all.mjs" }, "devDependencies": { "esbuild": "^0.28.0", diff --git a/packages/anastasis-cli/package.json b/packages/anastasis-cli/package.json @@ -18,8 +18,9 @@ "scripts": { "build": "tsc && ./build-node.mjs", "build:with-deps": "pnpm --filter \"{.}...\" run build", - "test": "tsc", - "coverage": "tsc && c8 ./bin/anastasis-cli.mjs", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc", + "coverage": "pnpm run test:clean && tsc && c8 ./bin/anastasis-cli.mjs", "clean": "rm -rf lib dist tsconfig.tsbuildinfo", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", "pretty": "prettier --write src" diff --git a/packages/anastasis-core/package.json b/packages/anastasis-core/package.json @@ -9,8 +9,9 @@ "build": "tsc", "build:with-deps": "pnpm --filter \"{.}...\" run build", "pretty": "prettier --write src", - "test": "tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", - "coverage": "tsc && c8 --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "coverage": "pnpm run test:clean && tsc && c8 --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", "clean": "rm -rf dist lib tsconfig.tsbuildinfo" }, diff --git a/packages/anastasis-webui/package.json b/packages/anastasis-webui/package.json @@ -11,7 +11,7 @@ "clean": "rm -rf dist lib tsconfig.tsbuildinfo", "lint": "../qa-tooling/bin/eslint.mjs .", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", - "test": "./test.mjs && node --test --enable-source-maps 'dist/**/*test.js'", + "test": "./test.mjs && node --test --enable-source-maps 'dist/test/**/*test.js'", "pretty": "prettier --write src" }, "dependencies": { diff --git a/packages/anastasis-webui/test.mjs b/packages/anastasis-webui/test.mjs @@ -23,8 +23,8 @@ await build({ type: "test", source: { js: allTestFiles.files, - assets: [{base:"src",files:["src/index.html"]}], + assets: [{ base: "src", files: ["src/index.html"] }], }, - destination: "./dist/prod", + destination: "./dist/test", css: "sass", }); diff --git a/packages/idb-bridge/package.json b/packages/idb-bridge/package.json @@ -10,7 +10,8 @@ "license": "AGPL-3.0-or-later", "private": false, "scripts": { - "test": "tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", "build": "tsc", "build:with-deps": "pnpm --filter \"{.}...\" run build", diff --git a/packages/libeufin-bank-webui/test.mjs b/packages/libeufin-bank-webui/test.mjs @@ -17,11 +17,9 @@ import { build } from "@gnu-taler/web-util/build"; import { getFilesInDirectory } from "@gnu-taler/web-util/build"; -import { rmSync } from "node:fs"; const allTestFiles = getFilesInDirectory("src", /\.test\.tsx?$/); -rmSync("./dist/test", { recursive: true, force: true }); await build({ type: "test", source: { diff --git a/packages/pogen/package.json b/packages/pogen/package.json @@ -8,7 +8,8 @@ "license": "GPL-2.0+", "scripts": { "clean": "rm -rf lib", - "test": "tsc6 && node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc6 && node --test 'lib/*.test.js' 'lib/**/*.test.js'", "build": "tsc6", "build:with-deps": "pnpm --filter \"{.}...\" run build" }, diff --git a/packages/qa-tooling/bin/test-all.mjs b/packages/qa-tooling/bin/test-all.mjs @@ -0,0 +1,243 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const scriptDirectory = path.dirname(fileURLToPath(import.meta.url)); +const repositoryRoot = path.resolve(scriptDirectory, "../../.."); +const reporterUrl = pathToFileURL( + path.join(scriptDirectory, "test-reporter.mjs"), +).href; + +function readJson(filename) { + return JSON.parse(readFileSync(filename, "utf8")); +} + +export function discoverTestPackages(root = repositoryRoot) { + const packagesDirectory = path.join(root, "packages"); + return readdirSync(packagesDirectory) + .map((entry) => path.join(packagesDirectory, entry)) + .filter((directory) => statSync(directory).isDirectory()) + .filter((directory) => existsSync(path.join(directory, "package.json"))) + .map((directory) => { + const manifest = readJson(path.join(directory, "package.json")); + return { directory, manifest, name: manifest.name }; + }) + .filter( + (pkg) => + pkg.name?.startsWith("@gnu-taler/") && + typeof pkg.manifest.scripts?.test === "string", + ); +} + +export function orderTestPackages(packages) { + const byName = new Map(packages.map((pkg) => [pkg.name, pkg])); + const permanent = new Set(); + const temporary = new Set(); + const ordered = []; + + function visit(pkg) { + if (permanent.has(pkg.name)) { + return; + } + if (temporary.has(pkg.name)) { + throw Error(`workspace dependency cycle involving ${pkg.name}`); + } + temporary.add(pkg.name); + const dependencyNames = Object.keys({ + ...pkg.manifest.dependencies, + ...pkg.manifest.optionalDependencies, + ...pkg.manifest.devDependencies, + }) + .filter((name) => byName.has(name)) + .sort(); + for (const dependencyName of dependencyNames) { + visit(byName.get(dependencyName)); + } + temporary.delete(pkg.name); + permanent.add(pkg.name); + ordered.push(pkg); + } + + for (const pkg of [...packages].sort((a, b) => + a.name.localeCompare(b.name), + )) { + visit(pkg); + } + return ordered; +} + +function emptyCounts() { + return { + tests: 0, + passed: 0, + failed: 0, + skipped: 0, + todo: 0, + cancelled: 0, + }; +} + +export function aggregateReports(reportDirectory, packageName) { + const counts = emptyCounts(); + for (const filename of readdirSync(reportDirectory)) { + if (!filename.endsWith(".json")) { + continue; + } + const report = readJson(path.join(reportDirectory, filename)); + if (report.packageName !== packageName) { + continue; + } + for (const key of Object.keys(counts)) { + counts[key] += report.counts[key] ?? 0; + } + } + return counts; +} + +function formatDuration(durationMs) { + if (durationMs < 1000) { + return `${durationMs}ms`; + } + return `${(durationMs / 1000).toFixed(1)}s`; +} + +function printSummary(results) { + const packageWidth = Math.max( + "Package".length, + ...results.map((result) => result.name.length), + ); + console.log("\nTest summary\n"); + console.log( + `${"Package".padEnd(packageWidth)} Status Tests Failed Duration`, + ); + console.log(`${"-".repeat(packageWidth)} ------ ----- ------ --------`); + for (const result of results) { + const testCount = result.counts.tests || "-"; + console.log( + `${result.name.padEnd(packageWidth)} ${result.ok ? "PASS" : "FAIL"} ${String(testCount).padStart(5)} ${String(result.counts.failed).padStart(6)} ${formatDuration(result.durationMs).padStart(8)}`, + ); + } + + const totals = results.reduce( + (acc, result) => { + acc.packagesPassed += result.ok ? 1 : 0; + acc.packagesFailed += result.ok ? 0 : 1; + acc.durationMs += result.durationMs; + for (const key of Object.keys(acc.counts)) { + acc.counts[key] += result.counts[key]; + } + return acc; + }, + { + packagesPassed: 0, + packagesFailed: 0, + durationMs: 0, + counts: emptyCounts(), + }, + ); + const modifiers = [ + totals.counts.skipped && `${totals.counts.skipped} skipped`, + totals.counts.todo && `${totals.counts.todo} todo`, + totals.counts.cancelled && `${totals.counts.cancelled} cancelled`, + ].filter(Boolean); + console.log( + `\nPackages: ${totals.packagesPassed} passed, ${totals.packagesFailed} failed; ` + + `tests: ${totals.counts.passed} passed, ${totals.counts.failed} failed` + + (modifiers.length ? `, ${modifiers.join(", ")}` : "") + + `; duration: ${formatDuration(totals.durationMs)}`, + ); +} + +function runPackage(pkg, reportDirectory, nodeOptions) { + return new Promise((resolve) => { + const started = Date.now(); + const child = spawn( + "pnpm", + ["--silent", "--dir", pkg.directory, "run", "test"], + { + cwd: repositoryRoot, + env: { + ...process.env, + NODE_OPTIONS: nodeOptions, + TALER_TEST_PACKAGE: pkg.name, + TALER_TEST_REPORT_DIR: reportDirectory, + }, + stdio: "inherit", + }, + ); + child.on("error", (error) => { + console.error(`failed to start ${pkg.name}: ${error.message}`); + resolve({ exitCode: 1, durationMs: Date.now() - started }); + }); + child.on("exit", (code, signal) => { + if (signal) { + console.error(`${pkg.name} terminated by ${signal}`); + } + resolve({ exitCode: code ?? 1, durationMs: Date.now() - started }); + }); + }); +} + +export async function main() { + const inheritedNodeOptions = process.env.NODE_OPTIONS ?? ""; + if (/(?:^|\s)--test-reporter(?:=|\s)/.test(inheritedNodeOptions)) { + throw Error( + "make check cannot combine its concise reporter with NODE_OPTIONS=--test-reporter", + ); + } + const nodeOptions = [inheritedNodeOptions, `--test-reporter=${reporterUrl}`] + .filter(Boolean) + .join(" "); + const reportDirectory = mkdtempSync(path.join(tmpdir(), "taler-tests-")); + const results = []; + + try { + const packages = orderTestPackages(discoverTestPackages()); + for (const pkg of packages) { + console.log(`\nRUN ${pkg.name}`); + const run = await runPackage(pkg, reportDirectory, nodeOptions); + const counts = aggregateReports(reportDirectory, pkg.name); + const ok = run.exitCode === 0; + const testDescription = counts.tests + ? `${counts.tests} ${counts.tests === 1 ? "test" : "tests"}` + : "compile only"; + console.log( + `${ok ? "PASS" : "FAIL"} ${pkg.name} (${testDescription}, ${formatDuration(run.durationMs)})`, + ); + if (!ok) { + console.log(`Rerun: pnpm --filter ${pkg.name} test`); + } + results.push({ + name: pkg.name, + ok, + durationMs: run.durationMs, + counts, + }); + } + printSummary(results); + process.exitCode = results.some((result) => !result.ok) ? 1 : 0; + } finally { + rmSync(reportDirectory, { recursive: true, force: true }); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main().catch((error) => { + console.error(error?.stack ?? error); + process.exitCode = 1; + }); +} diff --git a/packages/qa-tooling/bin/test-reporter.mjs b/packages/qa-tooling/bin/test-reporter.mjs @@ -0,0 +1,118 @@ +#!/usr/bin/env node + +import { mkdirSync, writeFileSync } from "node:fs"; +import { inspect } from "node:util"; +import path from "node:path"; + +function isSuite(data) { + return data.details?.type === "suite"; +} + +function isCancellation(data) { + const failureType = data.details?.error?.failureType; + return typeof failureType === "string" && /cancel|abort/i.test(failureType); +} + +function isParentFailure(data) { + return data.details?.error?.failureType === "subtestsFailed"; +} + +function formatLocation(data) { + if (!data.file) { + return "unknown location"; + } + const relativeFile = path.relative(process.cwd(), data.file) || data.file; + const line = data.line === undefined ? "" : `:${data.line}`; + const column = data.column === undefined ? "" : `:${data.column}`; + return `${relativeFile}${line}${column}`; +} + +function formatError(error) { + if (error?.cause && error.cause !== error) { + return formatError(error.cause); + } + if (typeof error?.stack === "string") { + return error.stack; + } + if (typeof error?.message === "string") { + return error.message; + } + return inspect(error, { colors: false, depth: 8 }); +} + +export function formatFailure(data) { + const detail = formatError(data.details?.error) + .split("\n") + .map((line) => ` ${line}`) + .join("\n"); + return `\nFAIL ${data.name}\n at ${formatLocation(data)}\n${detail}\n`; +} + +function writeSummary(counts) { + const reportDirectory = process.env.TALER_TEST_REPORT_DIR; + const packageName = process.env.TALER_TEST_PACKAGE; + if (!reportDirectory || !packageName) { + return; + } + mkdirSync(reportDirectory, { recursive: true }); + const safeName = packageName.replace(/[^a-zA-Z0-9_.-]+/g, "_"); + const filename = path.join( + reportDirectory, + `${safeName}-${process.pid}.json`, + ); + writeFileSync( + filename, + JSON.stringify({ packageName, pid: process.pid, counts }), + "utf8", + ); +} + +export async function* reportEvents(source, recordSummary) { + const counts = { + tests: 0, + passed: 0, + failed: 0, + skipped: 0, + todo: 0, + cancelled: 0, + }; + + for await (const event of source) { + if (event.type !== "test:pass" && event.type !== "test:fail") { + continue; + } + const data = event.data; + if (isSuite(data)) { + continue; + } + + counts.tests++; + if (data.todo) { + counts.todo++; + continue; + } + if (data.skip) { + counts.skipped++; + continue; + } + if (event.type === "test:pass") { + counts.passed++; + continue; + } + if (isCancellation(data)) { + counts.cancelled++; + continue; + } + + counts.failed++; + if (!isParentFailure(data)) { + yield formatFailure(data); + } + } + + recordSummary(counts); +} + +export default async function* conciseReporter(source) { + yield* reportEvents(source, writeSummary); +} diff --git a/packages/qa-tooling/package.json b/packages/qa-tooling/package.json @@ -4,6 +4,9 @@ "description": "QA tooling dependencies for GNU Taler", "private": true, "license": "AGPL-3.0-or-later", + "scripts": { + "test": "node --test 'test/*.test.mjs'" + }, "bin": { "eslint": "bin/eslint.mjs", "c8": "bin/c8.mjs" diff --git a/packages/qa-tooling/test/test-all.test.mjs b/packages/qa-tooling/test/test-all.test.mjs @@ -0,0 +1,134 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { aggregateReports, orderTestPackages } from "../bin/test-all.mjs"; +import { reportEvents } from "../bin/test-reporter.mjs"; + +function pkg(name, dependencies = {}) { + return { + directory: `/packages/${name}`, + manifest: { dependencies }, + name, + }; +} + +test("workspace dependencies run before their dependents", () => { + const packages = [ + pkg("@gnu-taler/application", { "@gnu-taler/library": "workspace:*" }), + pkg("@gnu-taler/unrelated"), + pkg("@gnu-taler/library"), + ]; + + assert.deepEqual( + orderTestPackages(packages).map((entry) => entry.name), + ["@gnu-taler/library", "@gnu-taler/application", "@gnu-taler/unrelated"], + ); +}); + +test("workspace dependency cycles are rejected", () => { + const packages = [ + pkg("@gnu-taler/first", { "@gnu-taler/second": "workspace:*" }), + pkg("@gnu-taler/second", { "@gnu-taler/first": "workspace:*" }), + ]; + + assert.throws( + () => orderTestPackages(packages), + /workspace dependency cycle/, + ); +}); + +test("reports from multiple test processes are aggregated", () => { + const directory = mkdtempSync(path.join(tmpdir(), "taler-test-report-")); + try { + writeFileSync( + path.join(directory, "one.json"), + JSON.stringify({ + packageName: "@gnu-taler/example", + counts: { tests: 3, passed: 2, failed: 1 }, + }), + ); + writeFileSync( + path.join(directory, "two.json"), + JSON.stringify({ + packageName: "@gnu-taler/example", + counts: { tests: 2, passed: 1, skipped: 1 }, + }), + ); + writeFileSync( + path.join(directory, "other-package.json"), + JSON.stringify({ + packageName: "@gnu-taler/other", + counts: { tests: 99, failed: 99 }, + }), + ); + + assert.deepEqual(aggregateReports(directory, "@gnu-taler/example"), { + tests: 5, + passed: 3, + failed: 1, + skipped: 1, + todo: 0, + cancelled: 0, + }); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("the concise reporter hides passes and retains failure details", async () => { + const assertion = new Error("expected the values to match"); + const events = [ + { + type: "test:pass", + data: { name: "passes", details: { type: "test" } }, + }, + { + type: "test:pass", + data: { + name: "skips", + skip: "not applicable", + details: { type: "test" }, + }, + }, + { + type: "test:pass", + data: { name: "suite", details: { type: "suite" } }, + }, + { + type: "test:fail", + data: { + name: "fails", + file: "/workspace/example.test.js", + line: 12, + column: 4, + details: { + type: "test", + error: { cause: assertion }, + }, + }, + }, + ]; + let summary; + const output = []; + + for await (const chunk of reportEvents(events, (counts) => { + summary = counts; + })) { + output.push(chunk); + } + + assert.equal(output.length, 1); + assert.match(output[0], /FAIL fails/); + assert.match(output[0], /example\.test\.js:12:4/); + assert.match(output[0], /expected the values to match/); + assert.deepEqual(summary, { + tests: 3, + passed: 1, + failed: 1, + skipped: 1, + todo: 0, + cancelled: 0, + }); +}); diff --git a/packages/qa-tooling/test/test-output-policy.test.mjs b/packages/qa-tooling/test/test-output-policy.test.mjs @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const repositoryRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../..", +); +const packagesDirectory = path.join(repositoryRoot, "packages"); + +test("TypeScript test scripts empty their compiler output first", () => { + const violations = []; + for (const entry of readdirSync(packagesDirectory, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const manifestFilename = path.join( + packagesDirectory, + entry.name, + "package.json", + ); + let manifest; + try { + manifest = JSON.parse(readFileSync(manifestFilename, "utf8")); + } catch (error) { + if (error.code === "ENOENT") { + continue; + } + throw error; + } + + for (const scriptName of ["test", "coverage", "coverage:html"]) { + const script = manifest.scripts?.[scriptName]; + if ( + typeof script === "string" && + /(?:^|&&\s*)(?:pnpm exec )?tsc6?(?:\s|&&|$)/.test(script) && + !script.includes("pnpm run test:clean") + ) { + violations.push(`${manifest.name}: ${scriptName}`); + } + } + } + + assert.deepEqual(violations, []); +}); + +test("esbuild test scripts do not emit into production directories", () => { + const violations = []; + for (const entry of readdirSync(packagesDirectory, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const testBuilder = path.join(packagesDirectory, entry.name, "test.mjs"); + let source; + try { + source = readFileSync(testBuilder, "utf8"); + } catch (error) { + if (error.code === "ENOENT") { + continue; + } + throw error; + } + if (/type:\s*["']test["']/.test(source) && /dist\/prod/.test(source)) { + violations.push(entry.name); + } + } + + assert.deepEqual(violations, []); +}); diff --git a/packages/taler-harness/package.json b/packages/taler-harness/package.json @@ -19,7 +19,8 @@ "build": "tsc && ./build.mjs", "build:with-deps": "pnpm --filter \"{.}...\" run build", "check": "tsc", - "test": "tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", "clean": "rm -rf lib dist tsconfig.tsbuildinfo", "pretty": "prettier --write src" diff --git a/packages/taler-util/package.json b/packages/taler-util/package.json @@ -70,8 +70,9 @@ "scripts": { "build": "tsc", "build:with-deps": "pnpm --filter \"{.}...\" run build", - "test": "tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", - "coverage": "tsc && c8 --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "coverage": "pnpm run test:clean && tsc && c8 --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", "clean": "rm -rf dist lib tsconfig.tsbuildinfo", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", "pretty": "prettier --write src" diff --git a/packages/taler-wallet-cli/package.json b/packages/taler-wallet-cli/package.json @@ -18,8 +18,9 @@ "scripts": { "build": "tsc && ./build-node.mjs", "build:with-deps": "pnpm --filter \"{.}...\" run build", - "test": "tsc && node --test lib/", - "coverage": "tsc && c8 ./bin/taler-wallet-cli.mjs", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && node --test lib/", + "coverage": "pnpm run test:clean && tsc && c8 ./bin/taler-wallet-cli.mjs", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", "clean": "rm -rf lib dist tsconfig.tsbuildinfo", "pretty": "prettier --write src" diff --git a/packages/taler-wallet-core/package.json b/packages/taler-wallet-core/package.json @@ -15,10 +15,11 @@ "build": "tsc", "build:with-deps": "pnpm --filter \"{.}...\" run build", "pretty": "prettier --write src", - "test": "tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && node --test 'lib/*.test.js' 'lib/**/*.test.js'", "typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/", - "coverage": "tsc && c8 --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", - "coverage:html": "tsc && c8 -r html --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "coverage": "pnpm run test:clean && tsc && c8 --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", + "coverage:html": "pnpm run test:clean && tsc && c8 -r html --src src --all node --test 'lib/*.test.js' 'lib/**/*.test.js'", "clean": "rm -rf dist lib tsconfig.tsbuildinfo" }, "files": [ diff --git a/packages/wallet-webui/package.json b/packages/wallet-webui/package.json @@ -23,7 +23,8 @@ "package:firefox": "node pack.mjs --target firefox", "package:extensions": "node pack.mjs --target all", "typecheck": "tsc --build --pretty false", - "test": "tsc --build --pretty false && node --test 'lib/test/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc --build --pretty false && node --test 'lib/test/**/*.test.js'", "test:boundaries": "node test/check-screen-boundaries.mjs", "lint:firefox": "web-ext lint --source-dir dist/extension/firefox --boring", "lint": "../qa-tooling/bin/eslint.mjs .", diff --git a/packages/web-util/package.json b/packages/web-util/package.json @@ -27,7 +27,8 @@ "scripts": { "build": "pnpm run clean && tsc && ./build.mjs", "build:with-deps": "pnpm --filter \"{.}...\" run build", - "test": "tsc && node --test 'lib/**/*.test.js'", + "test:clean": "rm -rf lib tsconfig.tsbuildinfo", + "test": "pnpm run test:clean && tsc && ./build.mjs && node --test 'lib/**/*.test.js'", "i18n:source2po": "pogen extract && pogen merge", "i18n:po2strings": "pogen emit", "clean": "rm -rf dist lib tsconfig.tsbuildinfo", diff --git a/packages/web-util/src/index.build.test.ts b/packages/web-util/src/index.build.test.ts @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { emptyTestOutputDirectory } from "./index.build.js"; + +test("test output is emptied without deleting the package root", () => { + const packageRoot = fs.mkdtempSync(path.join(os.tmpdir(), "web-util-test-")); + const outputDirectory = path.join(packageRoot, "dist/test"); + try { + fs.mkdirSync(outputDirectory, { recursive: true }); + fs.writeFileSync(path.join(outputDirectory, "stale.test.js"), "stale"); + + emptyTestOutputDirectory(packageRoot, "dist/test"); + + assert.equal(fs.existsSync(outputDirectory), false); + assert.equal(fs.existsSync(packageRoot), true); + } finally { + fs.rmSync(packageRoot, { recursive: true, force: true }); + } +}); + +test("test output cleanup rejects unsafe directories", () => { + const packageRoot = fs.mkdtempSync(path.join(os.tmpdir(), "web-util-test-")); + try { + assert.throws(() => emptyTestOutputDirectory(packageRoot, ".")); + assert.throws(() => emptyTestOutputDirectory(packageRoot, "../outside")); + } finally { + fs.rmSync(packageRoot, { recursive: true, force: true }); + } +}); diff --git a/packages/web-util/src/index.build.ts b/packages/web-util/src/index.build.ts @@ -179,6 +179,31 @@ export interface BuildParams { css?: "sass" | "postcss"; } +/** + * Remove the output of a test build without allowing the package root (or a + * directory outside it) to become the deletion target. + */ +export function emptyTestOutputDirectory( + packageRoot: string, + outputDirectory: string, +): void { + const resolvedRoot = path.resolve(packageRoot); + const resolvedOutput = path.resolve(resolvedRoot, outputDirectory); + const relativeOutput = path.relative(resolvedRoot, resolvedOutput); + + if ( + relativeOutput === "" || + relativeOutput === ".." || + relativeOutput.startsWith(`..${path.sep}`) || + path.isAbsolute(relativeOutput) + ) { + throw Error( + `refusing to empty test output outside the package: ${resolvedOutput}`, + ); + } + fs.rmSync(resolvedOutput, { recursive: true, force: true }); +} + function getPackageAndGitRoot(meta: undefined | ImportMeta) { if (meta) { const root = nodePath.dirname(nodeUrl.fileURLToPath(meta.url)); @@ -272,6 +297,12 @@ export function computeConfig(params: BuildParams): { */ export async function build(config: BuildParams) { const { esBuildOptions, version } = computeConfig(config); + if (config.type === "test") { + emptyTestOutputDirectory( + esBuildOptions.absWorkingDir ?? process.cwd(), + esBuildOptions.outdir ?? config.destination, + ); + } const res = await esbuild.build(esBuildOptions); fs.writeFileSync(`${esBuildOptions.outdir}/version.txt`, version); if (res.metafile) {