commit ad7166000230515fcd1cd1ff996ef85d1dad5e6e
parent 1af26b5a46848ba6e13723641f7e3e37ec76b8d7
Author: Florian Dold <dold@taler.net>
Date: Thu, 20 Aug 2026 23:55:20 +0200
taler-harness: require bug URLs for todo tests
Diffstat:
7 files changed, 174 insertions(+), 9 deletions(-)
diff --git a/packages/taler-harness/package.json b/packages/taler-harness/package.json
@@ -19,7 +19,7 @@
"build": "tsc && ./build.mjs",
"build:with-deps": "pnpm --filter \"{.}...\" run build",
"check": "tsc",
- "test": "tsc",
+ "test": "tsc && node --test 'lib/**/*.test.js'",
"typedoc": "pnpm dlx typedoc --out dist/typedoc ./src/",
"clean": "rm -rf lib dist tsconfig.tsbuildinfo",
"pretty": "prettier --write src"
diff --git a/packages/taler-harness/src/harness/harness.ts b/packages/taler-harness/src/harness/harness.ts
@@ -2789,6 +2789,11 @@ export interface TestRunResult {
*/
todo?: boolean;
+ /**
+ * Bug tracker URL explaining why the test is marked as "todo".
+ */
+ todoBugUrl?: string;
+
reason?: string;
}
diff --git a/packages/taler-harness/src/index.ts b/packages/taler-harness/src/index.ts
@@ -1929,7 +1929,7 @@ talerHarnessCli
s += ` [experimental]`;
}
if (t.todo) {
- s += ` [todo]`;
+ s += ` [todo: ${t.todoBugUrl}]`;
}
console.log(s);
}
diff --git a/packages/taler-harness/src/integrationtests/test-revocation.ts b/packages/taler-harness/src/integrationtests/test-revocation.ts
@@ -300,4 +300,4 @@ runRevocationTest.experimental = true;
// /recoup-refresh handlers disabled until exchange issue #9828 is resolved.
// Keep the scenario visible without treating that known protocol gap as a
// harness regression.
-runRevocationTest.todo = true;
+runRevocationTest.todo = "https://bugs.taler.net/n/9828";
diff --git a/packages/taler-harness/src/integrationtests/testrunner.ts b/packages/taler-harness/src/integrationtests/testrunner.ts
@@ -253,6 +253,7 @@ import { runWithdrawalIdempotentTest } from "./test-withdrawal-idempotent.js";
import { runWithdrawalManualTest } from "./test-withdrawal-manual.js";
import { runWithdrawalPrepareTest } from "./test-withdrawal-prepare.js";
import { runWithdrawalShortenTest } from "./test-withdrawal-shorten.js";
+import { TodoBugUrl, validateTodoBugUrl } from "./todo.js";
/**
* Test runner.
@@ -270,8 +271,10 @@ interface TestMainFunction {
* Mark the test as "todo": it is always run and reported, but a failure
* is tolerated and does not make the whole test run fail. Use this for
* tests that describe behavior that is not implemented (correctly) yet.
+ * The value must be a canonical Taler bug tracker URL of the form
+ * https://bugs.taler.net/n/$NUMBER.
*/
- todo?: boolean;
+ todo?: TodoBugUrl;
suites?: string[];
}
@@ -520,6 +523,7 @@ export interface TestInfo {
suites: string[];
experimental: boolean;
todo: boolean;
+ todoBugUrl?: string;
}
function updateCurrentSymlink(testDir: string): void {
@@ -552,12 +556,22 @@ export function getTestName(tf: TestMainFunction): string {
.toLowerCase();
}
+function validateTestMetadata(testCases: TestMainFunction[]): void {
+ for (const testCase of testCases) {
+ if (testCase.todo !== undefined) {
+ validateTodoBugUrl(testCase.todo, getTestName(testCase));
+ }
+ }
+}
+
interface RunTestChildInstruction {
testName: string;
testRootDir: string;
}
export async function runTests(spec: TestRunSpec) {
+ validateTestMetadata(allTests);
+
let testRootDir: string;
if (spec.testDir != null) {
testRootDir = spec.testDir;
@@ -641,10 +655,11 @@ export async function runTests(spec: TestRunSpec) {
for (const [n, testCase] of filteredTests.entries()) {
const testName = getTestName(testCase);
- const isTodo = testCase.todo ?? false;
+ const todoBugUrl = testCase.todo;
+ const isTodo = todoBugUrl !== undefined;
if (spec.dryRun) {
console.log(
- `dry run: would run test ${testName}${isTodo ? " (todo)" : ""}`,
+ `dry run: would run test ${testName}${isTodo ? ` (todo: ${todoBugUrl})` : ""}`,
);
continue;
}
@@ -699,7 +714,7 @@ export async function runTests(spec: TestRunSpec) {
progressText = progressText + `, todo failed ${numTodoFailed}`;
}
- const todoText = isTodo ? " [todo]" : "";
+ const todoText = isTodo ? ` [todo: ${todoBugUrl}]` : "";
if (spec.noTimeout) {
console.log(
@@ -782,6 +797,7 @@ export async function runTests(spec: TestRunSpec) {
if (isTodo) {
result.todo = true;
+ result.todoBugUrl = todoBugUrl;
}
// A failing todo test is expected, unless we're asked to be strict.
@@ -878,7 +894,11 @@ export function reportAndQuit(
console.log(`Passed: ${numPass}/${numTotal}`);
if (todoResults.length > 0) {
- // Which todo test had which result is in results.json.
+ for (const result of todoResults) {
+ console.log(
+ `Todo ${result.status}: ${result.name} (${result.todoBugUrl})`,
+ );
+ }
const numTodoFail = todoResults.filter((x) => x.status === "fail").length;
const suffix = strictTodo ? " (counted, --strict-todo)" : " (not counted)";
console.log(`Todo failed${suffix}: ${numTodoFail}/${todoResults.length}`);
@@ -894,11 +914,13 @@ export function reportAndQuit(
}
export function getTestInfo(): TestInfo[] {
+ validateTestMetadata(allTests);
return allTests.map((x) => ({
name: getTestName(x),
suites: x.suites ?? [],
experimental: x.experimental ?? false,
- todo: x.todo ?? false,
+ todo: x.todo !== undefined,
+ todoBugUrl: x.todo,
}));
}
diff --git a/packages/taler-harness/src/integrationtests/todo.test.ts b/packages/taler-harness/src/integrationtests/todo.test.ts
@@ -0,0 +1,104 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import assert from "node:assert";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { spawnSync } from "node:child_process";
+import { test } from "node:test";
+import { getTestInfo } from "./testrunner.js";
+import { TodoBugUrl, validateTodoBugUrl } from "./todo.js";
+
+test("accepts canonical Taler bug URLs", () => {
+ const value: unknown = "https://bugs.taler.net/n/9828";
+ validateTodoBugUrl(value, "revocation");
+ const bugUrl: TodoBugUrl = value;
+ assert.equal(bugUrl, "https://bugs.taler.net/n/9828");
+});
+
+test("rejects non-canonical todo bug URLs", () => {
+ const invalidValues: unknown[] = [
+ true,
+ "",
+ "http://bugs.taler.net/n/9828",
+ "https://bugs.example/n/9828",
+ "https://bugs.taler.net/9828",
+ "https://bugs.taler.net/n/",
+ "https://bugs.taler.net/n/not-a-number",
+ "https://bugs.taler.net/n/0",
+ "https://bugs.taler.net/n/9828/",
+ "https://bugs.taler.net/n/9828?view=full",
+ ];
+
+ for (const value of invalidValues) {
+ assert.throws(
+ () => validateTodoBugUrl(value, "example"),
+ /invalid todo bug URL for test example/,
+ );
+ }
+});
+
+test("exposes the bug URL in todo test information", () => {
+ const revocationTest = getTestInfo().find((x) => x.name === "revocation");
+ assert.deepEqual(revocationTest, {
+ name: "revocation",
+ suites: ["wallet"],
+ experimental: true,
+ todo: true,
+ todoBugUrl: "https://bugs.taler.net/n/9828",
+ });
+});
+
+test("writes and reports the bug URL for todo results", () => {
+ const testDir = fs.mkdtempSync(
+ path.join(os.tmpdir(), "taler-harness-todo-test-"),
+ );
+ const testRunnerUrl = new URL("./testrunner.js", import.meta.url).href;
+ const script = `
+ import { reportAndQuit } from ${JSON.stringify(testRunnerUrl)};
+ reportAndQuit(${JSON.stringify(testDir)}, [{
+ name: "example",
+ timeSec: 1,
+ status: "fail",
+ todo: true,
+ todoBugUrl: "https://bugs.taler.net/n/1234",
+ }]);
+ `;
+
+ try {
+ const child = spawnSync(
+ process.execPath,
+ ["--input-type=module", "--eval", script],
+ { encoding: "utf-8" },
+ );
+ assert.equal(child.status, 0, child.stderr);
+ assert.match(
+ child.stdout,
+ /Todo fail: example \(https:\/\/bugs\.taler\.net\/n\/1234\)/,
+ );
+
+ const results = JSON.parse(
+ fs.readFileSync(path.join(testDir, "results.json"), "utf-8"),
+ );
+ assert.equal(
+ results.testResults[0].todoBugUrl,
+ "https://bugs.taler.net/n/1234",
+ );
+ } finally {
+ fs.rmSync(testDir, { recursive: true, force: true });
+ }
+});
diff --git a/packages/taler-harness/src/integrationtests/todo.ts b/packages/taler-harness/src/integrationtests/todo.ts
@@ -0,0 +1,34 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+export type TodoBugUrl = string;
+
+const todoBugUrlPattern = /^https:\/\/bugs\.taler\.net\/n\/[1-9][0-9]*$/;
+
+/**
+ * Check that a todo test references a canonical Taler bug tracker URL.
+ */
+export function validateTodoBugUrl(
+ value: unknown,
+ testName: string,
+): asserts value is TodoBugUrl {
+ if (typeof value !== "string" || !todoBugUrlPattern.test(value)) {
+ throw Error(
+ `invalid todo bug URL for test ${testName}: expected ` +
+ `https://bugs.taler.net/n/$NUMBER, got ${JSON.stringify(value)}`,
+ );
+ }
+}
