diff options
Diffstat (limited to 'packages/taler-harness/src/integrationtests/testrunner.ts')
| -rw-r--r-- | packages/taler-harness/src/integrationtests/testrunner.ts | 597 |
1 files changed, 597 insertions, 0 deletions
diff --git a/packages/taler-harness/src/integrationtests/testrunner.ts b/packages/taler-harness/src/integrationtests/testrunner.ts new file mode 100644 index 000000000..4b23d7762 --- /dev/null +++ b/packages/taler-harness/src/integrationtests/testrunner.ts @@ -0,0 +1,597 @@ +/* + This file is part of GNU Taler + (C) 2021 Taler Systems S.A. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> + */ + +import { CancellationToken, Logger, minimatch } from "@gnu-taler/taler-util"; +import * as child_process from "child_process"; +import { spawnSync } from "child_process"; +import * as fs from "fs"; +import * as os from "os"; +import * as path from "path"; +import url from "url"; +import { + GlobalTestState, + TestRunResult, + runTestWithState, + shouldLingerInTest, +} from "../harness/harness.js"; +import { getSharedTestDir } from "../harness/helpers.js"; +import { runAgeRestrictionsDepositTest } from "./test-age-restrictions-deposit.js"; +import { runAgeRestrictionsMerchantTest } from "./test-age-restrictions-merchant.js"; +import { runAgeRestrictionsMixedMerchantTest } from "./test-age-restrictions-mixed-merchant.js"; +import { runAgeRestrictionsPeerTest } from "./test-age-restrictions-peer.js"; +import { runBankApiTest } from "./test-bank-api.js"; +import { runClaimLoopTest } from "./test-claim-loop.js"; +import { runClauseSchnorrTest } from "./test-clause-schnorr.js"; +import { runCurrencyScopeTest } from "./test-currency-scope.js"; +import { runDenomLostTest } from "./test-denom-lost.js"; +import { runDenomUnofferedTest } from "./test-denom-unoffered.js"; +import { runDepositTest } from "./test-deposit.js"; +import { runExchangeDepositTest } from "./test-exchange-deposit.js"; +import { runExchangeManagementFaultTest } from "./test-exchange-management-fault.js"; +import { runExchangeManagementTest } from "./test-exchange-management.js"; +import { runExchangePurseTest } from "./test-exchange-purse.js"; +import { runExchangeTimetravelTest } from "./test-exchange-timetravel.js"; +import { runFeeRegressionTest } from "./test-fee-regression.js"; +import { runForcedSelectionTest } from "./test-forced-selection.js"; +import { runKycTest } from "./test-kyc.js"; +import { runLibeufinBankTest } from "./test-libeufin-bank.js"; +import { runMerchantExchangeConfusionTest } from "./test-merchant-exchange-confusion.js"; +import { runMerchantInstancesDeleteTest } from "./test-merchant-instances-delete.js"; +import { runMerchantInstancesUrlsTest } from "./test-merchant-instances-urls.js"; +import { runMerchantInstancesTest } from "./test-merchant-instances.js"; +import { runMerchantLongpollingTest } from "./test-merchant-longpolling.js"; +import { runMerchantRefundApiTest } from "./test-merchant-refund-api.js"; +import { runMerchantSpecPublicOrdersTest } from "./test-merchant-spec-public-orders.js"; +import { runMultiExchangeTest } from "./test-multiexchange.js"; +import { runOtpTest } from "./test-otp.js"; +import { runPayPaidTest } from "./test-pay-paid.js"; +import { runPaymentAbortTest } from "./test-payment-abort.js"; +import { runPaymentClaimTest } from "./test-payment-claim.js"; +import { runPaymentDeletedTest } from "./test-payment-deleted.js"; +import { runPaymentExpiredTest } from "./test-payment-expired.js"; +import { runPaymentFaultTest } from "./test-payment-fault.js"; +import { runPaymentForgettableTest } from "./test-payment-forgettable.js"; +import { runPaymentIdempotencyTest } from "./test-payment-idempotency.js"; +import { runPaymentMultipleTest } from "./test-payment-multiple.js"; +import { runPaymentShareTest } from "./test-payment-share.js"; +import { runPaymentTemplateTest } from "./test-payment-template.js"; +import { runPaymentTransientTest } from "./test-payment-transient.js"; +import { runPaymentZeroTest } from "./test-payment-zero.js"; +import { runPaymentTest } from "./test-payment.js"; +import { runPaywallFlowTest } from "./test-paywall-flow.js"; +import { runPeerPullLargeTest } from "./test-peer-pull-large.js"; +import { runPeerPushLargeTest } from "./test-peer-push-large.js"; +import { runPeerRepairTest } from "./test-peer-repair.js"; +import { runPeerToPeerPullTest } from "./test-peer-to-peer-pull.js"; +import { runPeerToPeerPushTest } from "./test-peer-to-peer-push.js"; +import { runRefundAutoTest } from "./test-refund-auto.js"; +import { runRefundGoneTest } from "./test-refund-gone.js"; +import { runRefundIncrementalTest } from "./test-refund-incremental.js"; +import { runRefundTest } from "./test-refund.js"; +import { runRevocationTest } from "./test-revocation.js"; +import { runSimplePaymentTest } from "./test-simple-payment.js"; +import { runStoredBackupsTest } from "./test-stored-backups.js"; +import { runTimetravelAutorefreshTest } from "./test-timetravel-autorefresh.js"; +import { runTimetravelWithdrawTest } from "./test-timetravel-withdraw.js"; +import { runTermOfServiceFormatTest } from "./test-tos-format.js"; +import { runWalletBackupBasicTest } from "./test-wallet-backup-basic.js"; +import { runWalletBackupDoublespendTest } from "./test-wallet-backup-doublespend.js"; +import { runWalletBalanceNotificationsTest } from "./test-wallet-balance-notifications.js"; +import { runWalletBalanceZeroTest } from "./test-wallet-balance-zero.js"; +import { runWalletBalanceTest } from "./test-wallet-balance.js"; +import { runWalletBlockedDepositTest } from "./test-wallet-blocked-deposit.js"; +import { runWalletBlockedPayMerchantTest } from "./test-wallet-blocked-pay-merchant.js"; +import { runWalletBlockedPayPeerPullTest } from "./test-wallet-blocked-pay-peer-pull.js"; +import { runWalletBlockedPayPeerPushTest } from "./test-wallet-blocked-pay-peer-push.js"; +import { runWalletCliTerminationTest } from "./test-wallet-cli-termination.js"; +import { runWalletConfigTest } from "./test-wallet-config.js"; +import { runWalletCryptoWorkerTest } from "./test-wallet-cryptoworker.js"; +import { runWalletDblessTest } from "./test-wallet-dbless.js"; +import { runWalletDd48Test } from "./test-wallet-dd48.js"; +import { runWalletDenomExpireTest } from "./test-wallet-denom-expire.js"; +import { runWalletDevExperimentsTest } from "./test-wallet-dev-experiments.js"; +import { runWalletExchangeUpdateTest } from "./test-wallet-exchange-update.js"; +import { runWalletGenDbTest } from "./test-wallet-gendb.js"; +import { runWalletInsufficientBalanceTest } from "./test-wallet-insufficient-balance.js"; +import { runWalletNotificationsTest } from "./test-wallet-notifications.js"; +import { runWalletObservabilityTest } from "./test-wallet-observability.js"; +import { runWalletRefreshErrorsTest } from "./test-wallet-refresh-errors.js"; +import { runWalletRefreshTest } from "./test-wallet-refresh.js"; +import { runWalletWirefeesTest } from "./test-wallet-wirefees.js"; +import { runWallettestingTest } from "./test-wallettesting.js"; +import { runWithdrawalAbortBankTest } from "./test-withdrawal-abort-bank.js"; +import { runWithdrawalBankIntegratedTest } from "./test-withdrawal-bank-integrated.js"; +import { runWithdrawalConversionTest } from "./test-withdrawal-conversion.js"; +import { runWithdrawalFakebankTest } from "./test-withdrawal-fakebank.js"; +import { runWithdrawalFeesTest } from "./test-withdrawal-fees.js"; +import { runWithdrawalHandoverTest } from "./test-withdrawal-handover.js"; +import { runWithdrawalHugeTest } from "./test-withdrawal-huge.js"; +import { runWithdrawalManualTest } from "./test-withdrawal-manual.js"; + +/** + * Test runner. + */ +const logger = new Logger("testrunner.ts"); + +/** + * Spec for one test. + */ +interface TestMainFunction { + (t: GlobalTestState): Promise<void>; + timeoutMs?: number; + experimental?: boolean; + suites?: string[]; +} + +const allTests: TestMainFunction[] = [ + runAgeRestrictionsMerchantTest, + runAgeRestrictionsMixedMerchantTest, + runAgeRestrictionsPeerTest, + runAgeRestrictionsDepositTest, + runBankApiTest, + runClaimLoopTest, + runClauseSchnorrTest, + runDenomUnofferedTest, + runDepositTest, + runSimplePaymentTest, + runExchangeManagementFaultTest, + runExchangeTimetravelTest, + runFeeRegressionTest, + runForcedSelectionTest, + runKycTest, + runExchangePurseTest, + runExchangeDepositTest, + runMerchantExchangeConfusionTest, + runMerchantInstancesDeleteTest, + runMerchantInstancesTest, + runMerchantInstancesUrlsTest, + runMerchantLongpollingTest, + runMerchantRefundApiTest, + runMerchantSpecPublicOrdersTest, + runPaymentClaimTest, + runPaymentFaultTest, + runPaymentForgettableTest, + runPaymentIdempotencyTest, + runPaymentMultipleTest, + runPaymentTest, + runPaymentShareTest, + runPaymentTemplateTest, + runPaymentAbortTest, + runPaymentTransientTest, + runPaymentZeroTest, + runPayPaidTest, + runPeerRepairTest, + runMultiExchangeTest, + runWalletBalanceTest, + runPaywallFlowTest, + runPeerToPeerPullTest, + runPeerToPeerPushTest, + runRefundAutoTest, + runRefundGoneTest, + runRefundIncrementalTest, + runRefundTest, + runRevocationTest, + runWithdrawalManualTest, + runTimetravelAutorefreshTest, + runTimetravelWithdrawTest, + runWalletBackupBasicTest, + runWalletBackupDoublespendTest, + runWalletNotificationsTest, + runWalletCryptoWorkerTest, + runWalletDblessTest, + runWallettestingTest, + runWithdrawalAbortBankTest, + // runWithdrawalNotifyBeforeTxTest, + runWithdrawalBankIntegratedTest, + runWithdrawalFakebankTest, + runWithdrawalFeesTest, + runWithdrawalConversionTest, + runWithdrawalHugeTest, + runTermOfServiceFormatTest, + runStoredBackupsTest, + runPaymentExpiredTest, + runWalletGenDbTest, + runLibeufinBankTest, + runPaymentDeletedTest, + runWalletDd48Test, + runCurrencyScopeTest, + runWalletRefreshTest, + runWalletCliTerminationTest, + runOtpTest, + runWalletBalanceNotificationsTest, + runExchangeManagementTest, + runWalletConfigTest, + runWalletObservabilityTest, + runWalletDevExperimentsTest, + runWalletBalanceZeroTest, + runWalletInsufficientBalanceTest, + runWalletWirefeesTest, + runDenomLostTest, + runWalletDenomExpireTest, + runWalletBlockedDepositTest, + runWalletBlockedPayMerchantTest, + runWalletBlockedPayPeerPushTest, + runWalletBlockedPayPeerPullTest, + runWalletExchangeUpdateTest, + runWalletRefreshErrorsTest, + runPeerPullLargeTest, + runPeerPushLargeTest, + runWithdrawalHandoverTest, +]; + +export interface TestRunSpec { + includePattern?: string; + suiteSpec?: string; + dryRun?: boolean; + failFast?: boolean; + waitOnFail?: boolean; + includeExperimental: boolean; + noTimeout: boolean; + verbosity: number; +} + +export interface TestInfo { + name: string; + suites: string[]; + experimental: boolean; +} + +function updateCurrentSymlink(testDir: string): void { + const currLink = path.join( + os.tmpdir(), + `taler-integrationtests-${os.userInfo().username}-current`, + ); + try { + fs.unlinkSync(currLink); + } catch (e) { + // Ignore + } + try { + fs.symlinkSync(testDir, currLink); + } catch (e) { + console.log(e); + // Ignore + } +} + +export function getTestName(tf: TestMainFunction): string { + const res = tf.name.match(/run([a-zA-Z0-9]*)Test/); + if (!res) { + throw Error("invalid test name, must be 'run${NAME}Test'"); + } + return res[1] + .replace(/[a-z0-9][A-Z]/g, (x) => { + return x[0] + "-" + x[1]; + }) + .toLowerCase(); +} + +interface RunTestChildInstruction { + testName: string; + testRootDir: string; +} + +function purgeSharedTestEnvironment() { + const rmRes = spawnSync("rm", ["-rf", `${getSharedTestDir()}`]); + if (rmRes.status != 0) { + logger.warn("can't delete shared test directory"); + } + const psqlRes = spawnSync("psql", ["-Aqtl"], { + encoding: "utf-8", + }); + if (psqlRes.status != 0) { + logger.warn("could not list available postgres databases"); + return; + } + if (psqlRes.output[1]!!.indexOf("taler-integrationtest-shared") >= 0) { + const dropRes = spawnSync("dropdb", ["taler-integrationtest-shared"], { + encoding: "utf-8", + }); + if (dropRes.status != 0) { + logger.warn("could not drop taler-integrationtest-shared database"); + return; + } + } +} + +export async function runTests(spec: TestRunSpec) { + if (!process.env.TALER_HARNESS_KEEP) { + logger.info("purging shared test environment"); + purgeSharedTestEnvironment(); + } else { + logger.info("keeping shared test environment"); + } + + const testRootDir = fs.mkdtempSync( + path.join(os.tmpdir(), "taler-integrationtests-"), + ); + updateCurrentSymlink(testRootDir); + console.log(`testsuite root directory: ${testRootDir}`); + + const testResults: TestRunResult[] = []; + + let currentChild: child_process.ChildProcess | undefined; + + const handleSignal = (s: NodeJS.Signals) => { + console.log(`received signal ${s} in test parent`); + if (currentChild) { + currentChild.kill("SIGTERM"); + } + reportAndQuit(testRootDir, testResults, true); + }; + + process.on("SIGINT", (s) => handleSignal(s)); + process.on("SIGTERM", (s) => handleSignal(s)); + //process.on("unhandledRejection", handleSignal); + //process.on("uncaughtException", handleSignal); + + let suites: Set<string> | undefined; + + if (spec.suiteSpec) { + suites = new Set(spec.suiteSpec.split(",").map((x) => x.trim())); + } + + for (const [n, testCase] of allTests.entries()) { + const testName = getTestName(testCase); + if (spec.includePattern && !minimatch(testName, spec.includePattern)) { + continue; + } + + if (testCase.experimental && !spec.includeExperimental) { + continue; + } + + if (suites) { + const ts = new Set(testCase.suites ?? []); + const intersection = new Set([...suites].filter((x) => ts.has(x))); + if (intersection.size === 0) { + continue; + } + } + + if (spec.dryRun) { + console.log(`dry run: would run test ${testName}`); + continue; + } + + const testInstr: RunTestChildInstruction = { + testName, + testRootDir, + }; + + const myFilename = url.fileURLToPath(import.meta.url); + + currentChild = child_process.fork(myFilename, ["__TWCLI_TESTWORKER"], { + env: { + TWCLI_RUN_TEST_INSTRUCTION: JSON.stringify(testInstr), + ...process.env, + }, + stdio: ["pipe", "pipe", "pipe", "ipc"], + }); + + const testDir = path.join(testRootDir, testName); + fs.mkdirSync(testDir, { recursive: true }); + + const harnessLogFilename = path.join(testRootDir, testName, "harness.log"); + const harnessLogStream = fs.createWriteStream(harnessLogFilename); + + if (spec.verbosity > 0) { + currentChild.stderr?.pipe(process.stderr); + currentChild.stdout?.pipe(process.stdout); + } + + currentChild.stdout?.pipe(harnessLogStream); + currentChild.stderr?.pipe(harnessLogStream); + + // Default timeout when the test doesn't override it. + let defaultTimeout = 60000; + const overrideDefaultTimeout = process.env.TALER_TEST_TIMEOUT; + if (overrideDefaultTimeout) { + defaultTimeout = Number.parseInt(overrideDefaultTimeout, 10) * 1000; + } + + // Set the timeout to at least be the default timeout. + const testTimeoutMs = testCase.timeoutMs + ? Math.max(testCase.timeoutMs, defaultTimeout) + : defaultTimeout; + + if (spec.noTimeout) { + console.log(`running ${testName}, no timeout`); + } else { + console.log(`running ${testName} with timeout ${testTimeoutMs}ms`); + } + + const token = spec.noTimeout + ? CancellationToken.CONTINUE + : CancellationToken.timeout(testTimeoutMs).token; + + const resultPromise: Promise<TestRunResult> = new Promise( + (resolve, reject) => { + let msg: TestRunResult | undefined; + currentChild!.on("message", (m) => { + if (token.isCancelled) { + return; + } + msg = m as TestRunResult; + }); + currentChild!.on("exit", (code, signal) => { + if (token.isCancelled) { + return; + } + logger.info(`process exited code=${code} signal=${signal}`); + if (signal) { + reject(new Error(`test worker exited with signal ${signal}`)); + } else if (code != 0) { + reject(new Error(`test worker exited with code ${code}`)); + } else if (!msg) { + reject( + new Error( + `test worker exited without giving back the test results`, + ), + ); + } else { + resolve(msg); + } + }); + currentChild!.on("error", (err) => { + if (token.isCancelled) { + return; + } + reject(err); + }); + }, + ); + + let result: TestRunResult; + + try { + result = await token.racePromise(resultPromise); + if (result.status === "fail" && spec.failFast) { + logger.error("test failed and failing fast, exit!"); + throw Error("exit on fail fast"); + } + } catch (e: any) { + console.error(`test ${testName} timed out`); + if (token.isCancelled) { + result = { + status: "fail", + reason: "timeout", + timeSec: testTimeoutMs / 1000, + name: testName, + }; + currentChild.kill("SIGTERM"); + } else { + throw Error(e); + } + } + + harnessLogStream.close(); + + console.log(`parent: got result ${JSON.stringify(result)}`); + + testResults.push(result); + } + + reportAndQuit(testRootDir, testResults); +} + +export function reportAndQuit( + testRootDir: string, + testResults: TestRunResult[], + interrupted: boolean = false, +): never { + let numTotal = 0; + let numFail = 0; + let numSkip = 0; + let numPass = 0; + + for (const result of testResults) { + numTotal++; + if (result.status === "fail") { + numFail++; + } else if (result.status === "skip") { + numSkip++; + } else if (result.status === "pass") { + numPass++; + } + } + + const resultsFile = path.join(testRootDir, "results.json"); + fs.writeFileSync( + path.join(testRootDir, "results.json"), + JSON.stringify({ testResults, interrupted }, undefined, 2), + ); + if (interrupted) { + console.log("test suite was interrupted"); + } + console.log(`See ${resultsFile} for details`); + console.log(`Skipped: ${numSkip}/${numTotal}`); + console.log(`Failed: ${numFail}/${numTotal}`); + console.log(`Passed: ${numPass}/${numTotal}`); + + if (interrupted) { + process.exit(3); + } else if (numPass < numTotal - numSkip) { + process.exit(1); + } else { + process.exit(0); + } +} + +export function getTestInfo(): TestInfo[] { + return allTests.map((x) => ({ + name: getTestName(x), + suites: x.suites ?? [], + experimental: x.experimental ?? false, + })); +} + +const runTestInstrStr = process.env["TWCLI_RUN_TEST_INSTRUCTION"]; +if (runTestInstrStr && process.argv.includes("__TWCLI_TESTWORKER")) { + // Test will call taler-wallet-cli, so we must not propagate this variable. + delete process.env["TWCLI_RUN_TEST_INSTRUCTION"]; + const { testRootDir, testName } = JSON.parse( + runTestInstrStr, + ) as RunTestChildInstruction; + + process.on("disconnect", () => { + logger.trace("got disconnect from parent"); + process.exit(3); + }); + + const runTest = async () => { + let testMain: TestMainFunction | undefined; + for (const t of allTests) { + if (getTestName(t) === testName) { + testMain = t; + break; + } + } + + if (!process.send) { + logger.error("can't communicate with parent"); + process.exit(2); + } + + if (!testMain) { + logger.info(`test ${testName} not found`); + process.exit(2); + } + + const testDir = path.join(testRootDir, testName); + logger.info(`running test ${testName}`); + const gc = new GlobalTestState({ + testDir, + }); + const testResult = await runTestWithState(gc, testMain, testName); + logger.info(`done test ${testName}: ${testResult.status}`); + process.send(testResult); + }; + + runTest() + .then(() => { + logger.trace(`test ${testName} finished in worker`); + if (shouldLingerInTest()) { + logger.trace("lingering ..."); + return; + } + process.exit(0); + }) + .catch((e) => { + logger.error(e); + process.exit(1); + }); +} |