commit 2b6dfdeedd50787d9766690309ab93bbadba69ab parent fceb99fe6da2c92ba5069b63057f53aa064de29b Author: Florian Dold <dold@taler.net> Date: Fri, 7 Aug 2026 03:30:00 +0200 harness: give every integration test its own environment The shared environment restarted the in-memory fakebank while leaving the exchange's wirewatch progress in a database that outlived it, so every test after the first one to use it withdrew into a reserve that was never credited and hung until the runner killed it. It saved about a second per test on eight of them, which is not worth six tests that silently stopped testing anything. Diffstat:
12 files changed, 126 insertions(+), 288 deletions(-)
diff --git a/packages/taler-harness/README.md b/packages/taler-harness/README.md @@ -12,6 +12,75 @@ To get more actionable stack traces, enable source maps for node: export NODE_OPTIONS=--enable-source-maps ``` +## Integration tests + +One test per file in `src/integrationtests/`, each exporting `run<Name>Test`, +each registered in `testrunner.ts` — an unregistered test simply never runs. +The runner executes them **serially**: the services bind fixed ports (exchange +8081, bank 8082, merchant 8083) and share database names, so two runs on one +machine produce bogus failures in both. + +The harness runs from its bundle, not from `lib/`, so after editing anything +under `src/` you need: + +``` +./node_modules/.bin/tsc && ./build.mjs +``` + +Otherwise `list-integrationtests` keeps showing the old set and a glob naming a +new test reports "selected 0 tests". + +### Each test builds its own environment + +`createSimpleTestkudosEnvironmentV3` (and friends in `harness/environments.ts`) +give a test its own directory, its own database and its own service processes. +Nothing is carried between tests on purpose. + +There used to be a `useSharedTestkudosEnvironment` that reused one directory and +database across tests to save setup time. It was removed: it saved about a +second per test on a suite of eight, and it silently broke every test after the +first one to use it. The failure is worth understanding, because anything that +reintroduces shared state will reproduce it. + +The fakebank keeps its ledger in memory (`ram_limit` in `bank.conf`, no +database), so restarting it renumbers transactions from 1. The exchange, on a +database that outlived the restart, had already recorded +`work_shards.progress_row = 1` for its wirewatch job, and +`taler-exchange-wirewatch` asks the bank only for rows *after* that. The second +test's transfer therefore arrived as row 1, was never fetched, and the reserve +was never credited. The wallet then long-polled `/reserves/$RESERVE_PUB` +correctly and indefinitely until the runner killed the test. + +Two lifetimes have to agree: whatever resets the bank's ledger must also reset +the exchange's idea of how far it has read. + +### A test that only fails in a full run + +Almost always cross-test contamination rather than a real regression. Before +concluding anything, check that nothing is left over: + +``` +ss -ltn | grep 808 +``` + +Signatures worth recognising, all of which look like product bugs and are not: + +- `Failed to bind to port 8082: Address already in use` in `bank-stderr.log` — + a service from an earlier test is still up. Kill by pid; `pkill -f taler-...` + from a script also matches the script's own command line. +- `taler-exchange-offline: Fatal: exchange uses different master key!`, or + `ECONNRESET` during setup — the test is talking to *another* test's exchange. + A test that starts a second exchange on the shared port has to stop it again. +- A test that dies in well under a second usually means a missing binary rather + than a failing assertion. +- A test that hangs to its timeout with no assertion failure is waiting for + something that never happens. `harness.log` shows where it stopped; the wallet + daemon's `wallet-*-stderr.log` shows what it was polling for. + +Useful environment: `TALER_TEST_TIMEOUT`, `TALER_TEST_NO_TIMEOUT=1`, +`TALER_TEST_LINGER=1` to keep the services up after a test (it waits on stdin, +so hold that open), and `NODE_OPTIONS=--enable-source-maps`. + ## Headless Web Integration test 1) First you need the browsers that you are going to use to test diff --git a/packages/taler-harness/src/harness/environments.ts b/packages/taler-harness/src/harness/environments.ts @@ -89,7 +89,6 @@ import { MerchantServiceInterface, PaivanaService, setupDb, - setupSharedDb, useLibeufinBank, WalletCli, WalletClient, @@ -183,190 +182,6 @@ export interface EnvOptions { additionalBankConfig?(b: BankService): void; } -export function getSharedTestDir(): string { - return `/tmp/taler-harness@${process.env.USER}`; -} - -export async function useSharedTestkudosEnvironment(t: GlobalTestState) { - const coinConfig: CoinConfig[] = defaultCoinConfig.map((x) => x("TESTKUDOS")); - - const sharedDir = getSharedTestDir(); - - fs.mkdirSync(sharedDir, { recursive: true }); - - const db = await setupSharedDb(t); - - let bank: FakebankService; - - const prevSetupDone = fs.existsSync(sharedDir + "/setup-done"); - - logger.info(`previous setup done: ${prevSetupDone}`); - - // Wallet has longer startup-time and no dependencies, - // so we start it rather early. - const walletStartProm = createWalletDaemonWithClient(t, { name: "wallet" }); - - if (fs.existsSync(sharedDir + "/bank.conf")) { - logger.info("reusing existing bank"); - bank = FakebankService.fromExistingConfig(t, { - overridePath: sharedDir, - }); - } else { - logger.info("creating new bank config"); - bank = await FakebankService.create(t, { - allowRegistrations: true, - currency: "TESTKUDOS", - database: db.connStr, - httpPort: 8082, - overrideTestDir: sharedDir, - }); - } - - logger.info("setting up exchange"); - - const exchangeName = "testexchange-1"; - const exchangeConfigFilename = sharedDir + `/exchange-${exchangeName}.conf`; - - logger.info(`exchange config filename: ${exchangeConfigFilename}`); - - let exchange: ExchangeService; - - if (fs.existsSync(exchangeConfigFilename)) { - logger.info("reusing existing exchange config"); - exchange = ExchangeService.fromExistingConfig(t, exchangeName, { - overridePath: sharedDir, - }); - } else { - logger.info("creating new exchange config"); - exchange = ExchangeService.create(t, { - name: "testexchange-1", - currency: "TESTKUDOS", - httpPort: 8081, - database: db.connStr, - overrideTestDir: sharedDir, - }); - } - - logger.info("setting up merchant"); - - let merchant: MerchantService; - const merchantName = "testmerchant-1"; - const merchantConfigFilename = sharedDir + `/merchant-${merchantName}.conf`; - - if (fs.existsSync(merchantConfigFilename)) { - merchant = MerchantService.fromExistingConfig(t, merchantName, { - overridePath: sharedDir, - }); - } else { - merchant = await MerchantService.create(t, { - name: "testmerchant-1", - httpPort: 8083, - database: db.connStr, - overrideTestDir: sharedDir, - }); - } - - logger.info("creating bank account for exchange"); - - const exchangeBankAccount = await bank.createExchangeAccount( - "myexchange", - "password", - ); - - logger.info("creating exchange bank account"); - await exchange.addBankAccount("1", exchangeBankAccount); - - bank.setSuggestedExchange(exchange, exchangeBankAccount.accountPaytoUri); - - exchange.addCoinConfigList(coinConfig); - - merchant.addExchange(exchange); - - logger.info("basic setup done, starting services"); - - if (!prevSetupDone) { - // Must be done sequentially, due to a concurrency - // issue in the *-dbinit tools. - await exchange.dbinit(); - await merchant.dbinit(); - } - - await bank.start(); - - // We *must* wait for the exchange to be started. - // Otherwise the merchant might be able to see - // partial /keys (e.g. with missing wire fees), - // leading to flaky tests. - await exchange.start({ - skipDbinit: true, - skipKeyup: prevSetupDone, - }); - - const merchStart = async () => { - await merchant.start({ - skipDbinit: true, - }); - await merchant.pingUntilAvailable(); - - if (!prevSetupDone) { - const { accessToken: adminAccessToken } = - await merchant.addInstanceWithWireAccount({ - id: "admin", - name: "Default Instance", - paytoUris: [getTestHarnessPaytoForLabel("merchant-default")], - defaultWireTransferDelay: Duration.toTalerProtocolDuration( - Duration.fromSpec({ minutes: 1 }), - ), - }); - - await merchant.addInstanceWithWireAccount( - { - id: "minst1", - name: "minst1", - paytoUris: [getTestHarnessPaytoForLabel("minst1")], - defaultWireTransferDelay: Duration.toTalerProtocolDuration( - Duration.fromSpec({ minutes: 1 }), - ), - }, - { adminAccessToken }, - ); - } - - const merchApi = new TalerMerchantInstanceHttpClient( - merchant.makeInstanceBaseUrl(), - ); - const { access_token: merchantAdminAccessToken } = succeedOrThrow( - await merchApi.createAccessToken( - "admin", - MERCHANT_DEFAULT_AUTH.password, - MERCHANT_DEFAULT_LOGIN_SCOPE, - ), - ); - return { merchantAdminAccessToken }; - }; - - const res = await Promise.all([merchStart(), walletStartProm]); - - const merchantAdminAccessToken = res[0].merchantAdminAccessToken; - const walletClient = res[1].walletClient; - const walletService = res[1].walletService; - - fs.writeFileSync(sharedDir + "/setup-done", "OK"); - - logger.info("setup done!"); - - return { - commonDb: db, - exchange, - merchant, - merchantAdminAccessToken, - walletClient, - walletService, - bank, - exchangeBankAccount, - }; -} - /** * Run a test case with a simple TESTKUDOS Taler environment, consisting * of one exchange, one bank and one merchant. diff --git a/packages/taler-harness/src/harness/harness.ts b/packages/taler-harness/src/harness/harness.ts @@ -559,22 +559,6 @@ export async function setupDb( }; } -/** - * Make sure that the taler-integrationtest-shared database exists. - * Don't delete it if it already exists. - */ -export async function setupSharedDb(t: GlobalTestState): Promise<DbInfo> { - const dbname = "taler-integrationtest-shared"; - const databases = await runCommand(t, "list-dbs", "psql", ["-Aqtl"]); - if (databases.indexOf("taler-integrationtest-shared") < 0) { - await runCommand(t, "createdb", "createdb", [dbname]); - } - return { - connStr: `postgres:///${dbname}`, - dbname, - }; -} - export interface BankConfig { currency: string; httpPort: number; diff --git a/packages/taler-harness/src/integrationtests/test-known-accounts.ts b/packages/taler-harness/src/integrationtests/test-known-accounts.ts @@ -17,14 +17,10 @@ /** * Imports. */ -import { - j2s, - TalerCorebankApiClient, - TalerErrorCode, -} from "@gnu-taler/taler-util"; +import { j2s, TalerErrorCode } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState } from "../harness/harness.js"; @@ -35,10 +31,8 @@ import { GlobalTestState } from "../harness/harness.js"; export async function runKnownAccountsTest(t: GlobalTestState) { // Set up test environment - const { walletClient, bank, exchange } = - await useSharedTestkudosEnvironment(t); - - const bankClient = new TalerCorebankApiClient(bank.baseUrl); + const { walletClient, bank, exchange, bankClient } = + await createSimpleTestkudosEnvironmentV3(t); // Withdraw digital cash into the wallet. diff --git a/packages/taler-harness/src/integrationtests/test-merchant-reports.ts b/packages/taler-harness/src/integrationtests/test-merchant-reports.ts @@ -20,7 +20,6 @@ import { Duration, succeedOrThrow, - TalerCorebankApiClient, TalerMerchantApi, TalerMerchantInstanceHttpClient, TalerMerchantManagementHttpClient, @@ -30,7 +29,7 @@ import * as fs from "node:fs"; import { applyTimeTravelV2, makeTestPaymentV2, - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState, sh } from "../harness/harness.js"; @@ -47,10 +46,13 @@ export async function runMerchantReportsTest(t: GlobalTestState) { console.log(`XDG_DATA_HOME: ${process.env["XDG_DATA_HOME"]}`); - const { walletClient, bank, exchange, merchant, merchantAdminAccessToken } = - await useSharedTestkudosEnvironment(t); - - const bankClient = new TalerCorebankApiClient(bank.baseUrl); + const { + walletClient, + exchange, + merchant, + merchantAdminAccessToken, + bankClient, + } = await createSimpleTestkudosEnvironmentV3(t); const wres = await withdrawViaBankV3(t, { walletClient, diff --git a/packages/taler-harness/src/integrationtests/test-repurchase-v1.ts b/packages/taler-harness/src/integrationtests/test-repurchase-v1.ts @@ -21,7 +21,6 @@ import { ConfirmPayResultType, j2s, succeedOrThrow, - TalerCorebankApiClient, TalerMerchantInstanceHttpClient, TransactionMajorState, TransactionMinorState, @@ -29,7 +28,7 @@ import { } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState, harnessHttpLib } from "../harness/harness.js"; @@ -40,13 +39,16 @@ import { GlobalTestState, harnessHttpLib } from "../harness/harness.js"; export async function runRepurchaseV1Test(t: GlobalTestState) { // Set up test environment - const { walletClient, bank, exchange, merchant, merchantAdminAccessToken } = - await useSharedTestkudosEnvironment(t); + const { + walletClient, + exchange, + merchant, + merchantAdminAccessToken, + bankClient, + } = await createSimpleTestkudosEnvironmentV3(t); // Withdraw digital cash into the wallet. - const bankClient = new TalerCorebankApiClient(bank.baseUrl); - await withdrawViaBankV3(t, { walletClient, bankClient, diff --git a/packages/taler-harness/src/integrationtests/test-repurchase.ts b/packages/taler-harness/src/integrationtests/test-repurchase.ts @@ -21,7 +21,6 @@ import { ConfirmPayResultType, j2s, succeedOrThrow, - TalerCorebankApiClient, TalerMerchantInstanceHttpClient, TransactionMajorState, TransactionMinorState, @@ -29,7 +28,7 @@ import { } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState, harnessHttpLib } from "../harness/harness.js"; @@ -37,13 +36,16 @@ import { GlobalTestState, harnessHttpLib } from "../harness/harness.js"; export async function runRepurchaseTest(t: GlobalTestState) { // Set up test environment - const { walletClient, bank, exchange, merchant, merchantAdminAccessToken } = - await useSharedTestkudosEnvironment(t); + const { + walletClient, + exchange, + merchant, + merchantAdminAccessToken, + bankClient, + } = await createSimpleTestkudosEnvironmentV3(t); // Withdraw digital cash into the wallet. - const bankClient = new TalerCorebankApiClient(bank.baseUrl); - await withdrawViaBankV3(t, { walletClient, bankClient, diff --git a/packages/taler-harness/src/integrationtests/test-simple-payment.ts b/packages/taler-harness/src/integrationtests/test-simple-payment.ts @@ -17,14 +17,11 @@ /** * Imports. */ -import { - TalerCorebankApiClient, - TalerMerchantApi, -} from "@gnu-taler/taler-util"; +import { TalerMerchantApi } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { makeTestPaymentV2, - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState } from "../harness/harness.js"; @@ -35,11 +32,15 @@ import { GlobalTestState } from "../harness/harness.js"; export async function runSimplePaymentTest(t: GlobalTestState) { // Set up test environment - const { walletClient, bank, exchange, merchant, merchantAdminAccessToken } = - await useSharedTestkudosEnvironment(t); + const { + walletClient, + exchange, + merchant, + merchantAdminAccessToken, + bankClient, + } = await createSimpleTestkudosEnvironmentV3(t); // Withdraw digital cash into the wallet. - const bankClient = new TalerCorebankApiClient(bank.baseUrl); await withdrawViaBankV3(t, { walletClient, diff --git a/packages/taler-harness/src/integrationtests/test-stored-backups.ts b/packages/taler-harness/src/integrationtests/test-stored-backups.ts @@ -17,14 +17,11 @@ /** * Imports. */ -import { - TalerCorebankApiClient, - TalerMerchantApi, -} from "@gnu-taler/taler-util"; +import { TalerMerchantApi } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { makeTestPaymentV2, - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState } from "../harness/harness.js"; @@ -35,13 +32,16 @@ import { GlobalTestState } from "../harness/harness.js"; export async function runStoredBackupsTest(t: GlobalTestState) { // Set up test environment - const { walletClient, bank, exchange, merchant, merchantAdminAccessToken } = - await useSharedTestkudosEnvironment(t); + const { + walletClient, + exchange, + merchant, + merchantAdminAccessToken, + bankClient, + } = await createSimpleTestkudosEnvironmentV3(t); // Withdraw digital cash into the wallet. - const bankClient = new TalerCorebankApiClient(bank.baseUrl); - const wres = await withdrawViaBankV3(t, { walletClient, bankClient, diff --git a/packages/taler-harness/src/integrationtests/test-wallet-devexp-fakeprotover.ts b/packages/taler-harness/src/integrationtests/test-wallet-devexp-fakeprotover.ts @@ -21,13 +21,12 @@ import { LibtoolVersion, Logger, succeedOrThrow, - TalerCorebankApiClient, TalerErrorCode, TalerExchangeHttpClient, } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState, harnessHttpLib } from "../harness/harness.js"; @@ -35,8 +34,8 @@ import { GlobalTestState, harnessHttpLib } from "../harness/harness.js"; const logger = new Logger("test-wallet-devexp-fakeprotover.ts"); export async function runWalletDevexpFakeprotoverTest(t: GlobalTestState) { - const { walletClient, exchange, merchant, bank } = - await useSharedTestkudosEnvironment(t); + const { walletClient, exchange, merchant, bankClient } = + await createSimpleTestkudosEnvironmentV3(t); const exchangeClient = new TalerExchangeHttpClient(exchange.baseUrl, { httpClient: harnessHttpLib, @@ -47,8 +46,6 @@ export async function runWalletDevexpFakeprotoverTest(t: GlobalTestState) { const newerVer = `${exchVer.current + 1}:0:0`; - const bankClient = new TalerCorebankApiClient(bank.baseUrl); - const wres = await withdrawViaBankV3(t, { walletClient, exchange, diff --git a/packages/taler-harness/src/integrationtests/test-wallet-transactions.ts b/packages/taler-harness/src/integrationtests/test-wallet-transactions.ts @@ -22,14 +22,13 @@ import { Duration, GetTransactionsV2Request, j2s, - TalerCorebankApiClient, TalerMerchantApi, TransactionIdStr, } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { makeTestPaymentV2, - useSharedTestkudosEnvironment, + createSimpleTestkudosEnvironmentV3, withdrawViaBankV3, } from "../harness/environments.js"; import { GlobalTestState } from "../harness/harness.js"; @@ -37,8 +36,13 @@ import { GlobalTestState } from "../harness/harness.js"; export async function runWalletTransactionsTest(t: GlobalTestState) { // Set up test environment - const { walletClient, bank, exchange, merchant, merchantAdminAccessToken } = - await useSharedTestkudosEnvironment(t); + const { + walletClient, + exchange, + merchant, + merchantAdminAccessToken, + bankClient, + } = await createSimpleTestkudosEnvironmentV3(t); { const txRes = await walletClient.call( @@ -50,8 +54,6 @@ export async function runWalletTransactionsTest(t: GlobalTestState) { // Withdraw digital cash into the wallet. - const bankClient = new TalerCorebankApiClient(bank.baseUrl); - await withdrawViaBankV3(t, { walletClient, bankClient, diff --git a/packages/taler-harness/src/integrationtests/testrunner.ts b/packages/taler-harness/src/integrationtests/testrunner.ts @@ -26,7 +26,6 @@ import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import url from "node:url"; -import { getSharedTestDir } from "../harness/environments.js"; import { GlobalTestState, runTestWithState, @@ -562,36 +561,7 @@ interface RunTestChildInstruction { testRootDir: string; } -function purgeSharedTestEnvironment() { - const rmRes = spawnSync("rm", ["-rf", `${getSharedTestDir()}`]); - if (rmRes.status != 0) { - logger.warn("can't delete shared test directory"); - } - const psqlRes = spawnSync("psql", ["-Aqtl"], { - encoding: "utf-8", - }); - if (psqlRes.status != 0) { - logger.warn("could not list available postgres databases"); - return; - } - if (psqlRes.output[1]!.indexOf("taler-integrationtest-shared") >= 0) { - const dropRes = spawnSync("dropdb", ["taler-integrationtest-shared"], { - encoding: "utf-8", - }); - if (dropRes.status != 0) { - logger.warn("could not drop taler-integrationtest-shared database"); - return; - } - } -} - export async function runTests(spec: TestRunSpec) { - if (!process.env.TALER_HARNESS_KEEP) { - logger.info("purging shared test environment"); - purgeSharedTestEnvironment(); - } else { - logger.info("keeping shared test environment"); - } let testRootDir: string; if (spec.testDir != null) { testRootDir = spec.testDir;