commit c3fffe82bdf240a15b4576f3392cc27ca2794466
parent ad824ad9162076c6b1894bee4ecba05debd78d6b
Author: Florian Dold <dold@taler.net>
Date: Sun, 13 Sep 2026 20:52:03 +0200
taler-harness: allow suites to be disabled by default
Require explicit --suites selection for suites with additional setup.
Keep this independent of test name patterns and --experimental, and
show the disabled state when listing integration tests.
Diffstat:
6 files changed, 221 insertions(+), 48 deletions(-)
diff --git a/packages/taler-harness/README.md b/packages/taler-harness/README.md
@@ -4,6 +4,17 @@ This package implements the `taler-harness` CLI tool. It contains integration
tests for GNU Taler and GNU anastasis, as well as various helpers for managing
deployments of GNU Taler.
+## Integration test selection
+
+`run-integrationtests` runs enabled suites by default. Suites that require
+additional setup can be disabled by default in
+`src/integrationtests/test-selection.ts`; currently this applies to
+`wallet-android`. Enable such a suite with `--suites wallet-android`.
+A test-name pattern only narrows the selection and does not enable a disabled
+suite. `--experimental` controls experimental tests independently and also does
+not enable disabled suites. `list-integrationtests` marks tests belonging to a
+disabled suite, and `run-integrationtests --dry` shows the actual selection.
+
## Stagefright merchant browser scenarios
Run the full merchant scenario using this checkout's Web UI against staging:
diff --git a/packages/taler-harness/src/index.ts b/packages/taler-harness/src/index.ts
@@ -1984,6 +1984,9 @@ talerHarnessCli
if (t.experimental) {
s += ` [experimental]`;
}
+ if (t.disabledByDefault) {
+ s += ` [suite disabled by default]`;
+ }
if (t.todo) {
s += ` [todo: ${t.todoBugUrl}]`;
}
@@ -1997,7 +2000,7 @@ talerHarnessCli
help: "Glob pattern to select which tests to run",
})
.maybeOption("suites", ["--suites"], clk.STRING, {
- help: "Only run selected suites (comma-separated list)",
+ help: "Select suites, including suites disabled by default (comma-separated list)",
})
.maybeOption("testDir", ["--test-dir"], clk.STRING, {
help: "When to run the tests",
diff --git a/packages/taler-harness/src/integrationtests/test-selection.test.ts b/packages/taler-harness/src/integrationtests/test-selection.test.ts
@@ -0,0 +1,110 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { selectTests, TestSelectionSpec } from "./test-selection.js";
+
+const tests = [
+ { name: "unclassified" },
+ { name: "wallet-basic", suites: ["wallet"] },
+ { name: "wallet-experimental", suites: ["wallet"], experimental: true },
+ { name: "wallet-android-basic", suites: ["wallet-android"] },
+ { name: "wallet-android-shared", suites: ["wallet", "wallet-android"] },
+ {
+ name: "wallet-android-experimental",
+ suites: ["wallet-android"],
+ experimental: true,
+ },
+];
+
+const cases: {
+ name: string;
+ spec: Partial<TestSelectionSpec>;
+ expected: string[];
+}[] = [
+ {
+ name: "default selection excludes disabled suites and experimental tests",
+ spec: {},
+ expected: ["unclassified", "wallet-basic"],
+ },
+ {
+ name: "experimental flag does not enable disabled suites",
+ spec: { includeExperimental: true },
+ expected: ["unclassified", "wallet-basic", "wallet-experimental"],
+ },
+ {
+ name: "matching patterns do not enable disabled suites",
+ spec: { includePattern: "wallet-android-*", includeExperimental: true },
+ expected: [],
+ },
+ {
+ name: "explicit suite selection enables its ordinary tests",
+ spec: { suiteSpec: "wallet-android" },
+ expected: ["wallet-android-basic", "wallet-android-shared"],
+ },
+ {
+ name: "experimental tests in disabled suites still need both options",
+ spec: { suiteSpec: "wallet-android", includeExperimental: true },
+ expected: [
+ "wallet-android-basic",
+ "wallet-android-shared",
+ "wallet-android-experimental",
+ ],
+ },
+ {
+ name: "another suite cannot bypass the required opt-in",
+ spec: { suiteSpec: "wallet" },
+ expected: ["wallet-basic"],
+ },
+ {
+ name: "comma-separated suites support mixed enabled and disabled suites",
+ spec: { suiteSpec: "wallet, wallet-android" },
+ expected: ["wallet-basic", "wallet-android-basic", "wallet-android-shared"],
+ },
+ {
+ name: "patterns narrow an explicitly enabled suite",
+ spec: {
+ suiteSpec: "wallet-android",
+ includePattern: "wallet-android-basic",
+ },
+ expected: ["wallet-android-basic"],
+ },
+ {
+ name: "comma-separated patterns intersect suite selection",
+ spec: {
+ suiteSpec: "wallet-android",
+ includePattern: "wallet-basic,wallet-android-shared",
+ },
+ expected: ["wallet-android-shared"],
+ },
+ {
+ name: "unknown suites do not select any tests",
+ spec: { suiteSpec: "unknown" },
+ expected: [],
+ },
+];
+
+for (const { name, spec, expected } of cases) {
+ test(name, () => {
+ assert.deepEqual(
+ selectTests(tests, { includeExperimental: false, ...spec }).map(
+ (t) => t.name,
+ ),
+ expected,
+ );
+ });
+}
diff --git a/packages/taler-harness/src/integrationtests/test-selection.ts b/packages/taler-harness/src/integrationtests/test-selection.ts
@@ -0,0 +1,77 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import { minimatch } from "@gnu-taler/taler-util";
+
+export interface TestSelectionSpec {
+ includePattern?: string;
+ suiteSpec?: string;
+ includeExperimental: boolean;
+}
+
+interface SelectableTest {
+ name: string;
+ suites?: readonly string[];
+ experimental?: boolean;
+}
+
+// These suites require an explicit --suites entry, even when a test pattern
+// matches or --experimental is given. Other suites remain enabled by default.
+const disabledByDefaultSuites: ReadonlySet<string> = new Set([
+ "wallet-android",
+]);
+
+export function isTestDisabledByDefault(
+ test: Pick<SelectableTest, "suites">,
+): boolean {
+ return (
+ test.suites?.some((suite) => disabledByDefaultSuites.has(suite)) ?? false
+ );
+}
+
+export function selectTests<T extends SelectableTest>(
+ tests: readonly T[],
+ spec: TestSelectionSpec,
+): T[] {
+ const suites = spec.suiteSpec
+ ? new Set(spec.suiteSpec.split(",").map((suite) => suite.trim()))
+ : undefined;
+ const patterns = spec.includePattern?.split(",");
+
+ return tests.filter((test) => {
+ if (
+ patterns &&
+ !patterns.some((pattern) => minimatch(test.name, pattern))
+ ) {
+ return false;
+ }
+ if (test.experimental && !spec.includeExperimental) {
+ return false;
+ }
+ // Membership in an ordinary suite must not bypass the required opt-in.
+ if (
+ isTestDisabledByDefault(test) &&
+ !test.suites?.some(
+ (suite) => disabledByDefaultSuites.has(suite) && suites?.has(suite),
+ )
+ ) {
+ return false;
+ }
+ return (
+ !suites || (test.suites?.some((suite) => suites.has(suite)) ?? false)
+ );
+ });
+}
diff --git a/packages/taler-harness/src/integrationtests/testrunner.ts b/packages/taler-harness/src/integrationtests/testrunner.ts
@@ -15,11 +15,7 @@
*/
import { runTransactionFinalAmountsTest } from "./test-transaction-final-amounts.js";
-import {
- Logger,
- minimatch,
- setGlobalLogLevelFromString,
-} from "@gnu-taler/taler-util";
+import { Logger, setGlobalLogLevelFromString } from "@gnu-taler/taler-util";
import * as fs from "node:fs";
import * as os from "node:os";
import * as path from "node:path";
@@ -309,6 +305,11 @@ import { runWithdrawalManualTest } from "./test-withdrawal-manual.js";
import { runWithdrawalShortenTest } from "./test-withdrawal-shorten.js";
import { TodoBugUrl, validateTodoBugUrl } from "./todo.js";
import { withWalletDbBackend } from "./wallet-db-backends.js";
+import {
+ isTestDisabledByDefault,
+ selectTests,
+ TestSelectionSpec,
+} from "./test-selection.js";
/**
* Test runner.
@@ -590,16 +591,13 @@ const allTests: TestMainFunction[] = [
runMerchantOrderListingTest,
];
-export interface TestRunSpec {
+export interface TestRunSpec extends TestSelectionSpec {
/** Service log audit policy, defaulting to warn. */
logAudit?: LogAuditMode;
- includePattern?: string;
- suiteSpec?: string;
testDir?: string;
dryRun?: boolean;
failFast?: boolean;
waitOnFail?: boolean;
- includeExperimental: boolean;
/**
* Treat failures of todo tests as real failures.
*/
@@ -618,6 +616,7 @@ export interface TestInfo {
name: string;
suites: string[];
experimental: boolean;
+ disabledByDefault: boolean;
todo: boolean;
todoBugUrl?: string;
}
@@ -696,44 +695,15 @@ export async function runTests(spec: TestRunSpec) {
process.on("SIGINT", handleSignal);
process.on("SIGTERM", handleSignal);
- let suites: Set<string> | undefined;
-
- if (spec.suiteSpec) {
- suites = new Set(spec.suiteSpec.split(",").map((x) => x.trim()));
- }
-
- const filteredTests: TestMainFunction[] = [];
-
- const patterns = spec.includePattern?.split(",");
-
- for (const [, testCase] of allTests.entries()) {
- const testName = getTestName(testCase);
- if (patterns) {
- let matched = false;
- for (const pat of patterns) {
- if (minimatch(testName, pat)) {
- matched = true;
- break;
- }
- }
- if (!matched) {
- continue;
- }
- }
-
- if (testCase.experimental && !spec.includeExperimental) {
- continue;
- }
-
- if (suites) {
- const ts = new Set(testCase.suites ?? []);
- const intersection = new Set([...suites].filter((x) => ts.has(x)));
- if (intersection.size === 0) {
- continue;
- }
- }
- filteredTests.push(testCase);
- }
+ const filteredTests = selectTests(
+ allTests.map((test) => ({
+ name: getTestName(test),
+ suites: test.suites,
+ experimental: test.experimental,
+ test,
+ })),
+ spec,
+ ).map((entry) => entry.test);
console.log(`selected ${filteredTests.length} tests`);
@@ -1029,6 +999,7 @@ export function getTestInfo(): TestInfo[] {
name: getTestName(x),
suites: x.suites ?? [],
experimental: x.experimental ?? false,
+ disabledByDefault: isTestDisabledByDefault(x),
todo: x.todo !== undefined,
todoBugUrl: x.todo,
}));
diff --git a/packages/taler-harness/src/integrationtests/todo.test.ts b/packages/taler-harness/src/integrationtests/todo.test.ts
@@ -58,6 +58,7 @@ test("exposes the bug URL in todo test information", () => {
name: "revocation",
suites: ["wallet"],
experimental: true,
+ disabledByDefault: false,
todo: true,
todoBugUrl: "https://bugs.taler.net/n/9828",
});
