From 050999a910837f8a5353b1584af2b03bd8dad93d Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 13 Jan 2021 00:50:56 +0100 Subject: implement infrastructure for future DB migrations via backup --- .../src/integrationtests/harness.ts | 20 ++- .../src/integrationtests/testrunner.ts | 11 +- packages/taler-wallet-core/src/db.ts | 182 ++++++++++++++------- packages/taler-wallet-core/src/headless/helpers.ts | 5 +- packages/taler-wallet-core/src/index.ts | 1 + .../taler-wallet-core/src/operations/exchanges.ts | 1 + packages/taler-wallet-core/src/operations/state.ts | 3 +- packages/taler-wallet-core/src/types/dbTypes.ts | 15 +- packages/taler-wallet-core/src/util/query.ts | 29 ++-- packages/taler-wallet-core/src/wallet.ts | 7 +- .../taler-wallet-webextension/src/wxBackend.ts | 8 +- 11 files changed, 189 insertions(+), 93 deletions(-) (limited to 'packages') diff --git a/packages/taler-wallet-cli/src/integrationtests/harness.ts b/packages/taler-wallet-cli/src/integrationtests/harness.ts index 108b78540..4985c5fc1 100644 --- a/packages/taler-wallet-cli/src/integrationtests/harness.ts +++ b/packages/taler-wallet-cli/src/integrationtests/harness.ts @@ -78,6 +78,7 @@ import { AcceptTipRequest, AbortPayWithRefundRequest, handleWorkerError, + openPromise, } from "taler-wallet-core"; import { URL } from "url"; import axios, { AxiosError } from "axios"; @@ -94,7 +95,6 @@ import { import { ApplyRefundResponse } from "taler-wallet-core"; import { PendingOperationsResponse } from "taler-wallet-core"; import { CoinConfig } from "./denomStructures"; -import { after } from "taler-wallet-core/src/util/timer"; const exec = util.promisify(require("child_process").exec); @@ -1114,11 +1114,14 @@ export class ExchangeService implements ExchangeServiceInterface { `exchange-httpd-${this.name}`, ); + await this.pingUntilAvailable(); await this.keyup(); } async pingUntilAvailable(): Promise { - const url = `http://localhost:${this.exchangeConfig.httpPort}/keys`; + // We request /management/keys, since /keys can block + // when we didn't do the key setup yet. + const url = `http://localhost:${this.exchangeConfig.httpPort}/management/keys`; await pingProc(this.exchangeHttpProc, url, `exchange (${this.name})`); } } @@ -1449,10 +1452,14 @@ export async function runTestWithState( ): Promise { const startMs = new Date().getTime(); - const handleSignal = () => { + const p = openPromise(); + let status: TestStatus; + + const handleSignal = (s: string) => { gc.shutdownSync(); - console.warn("**** received fatal signal, shutting down test harness"); - process.exit(1); + console.warn("**** received fatal proces event, shutting down test harness"); + status = "fail"; + p.reject(Error("caught signal")); }; process.on("SIGINT", handleSignal); @@ -1460,10 +1467,9 @@ export async function runTestWithState( process.on("unhandledRejection", handleSignal); process.on("uncaughtException", handleSignal); - let status: TestStatus; try { console.log("running test in directory", gc.testDir); - await testMain(gc); + await Promise.race([testMain(gc), p.promise]); status = "pass"; } catch (e) { console.error("FATAL: test failed with exception", e); diff --git a/packages/taler-wallet-cli/src/integrationtests/testrunner.ts b/packages/taler-wallet-cli/src/integrationtests/testrunner.ts index d9804562e..578e9488c 100644 --- a/packages/taler-wallet-cli/src/integrationtests/testrunner.ts +++ b/packages/taler-wallet-cli/src/integrationtests/testrunner.ts @@ -66,12 +66,13 @@ const allTests: TestMainFunction[] = [ runMerchantLongpollingTest, runMerchantRefundApiTest, runPayAbortTest, - runPayPaidTest, runPaymentClaimTest, runPaymentFaultTest, runPaymentIdempotencyTest, runPaymentMultipleTest, + runPaymentTest, runPaymentTransientTest, + runPayPaidTest, runPaywallFlowTest, runRefundAutoTest, runRefundGoneTest, @@ -82,10 +83,9 @@ const allTests: TestMainFunction[] = [ runTimetravelWithdrawTest, runTippingTest, runWallettestingTest, + runTestWithdrawalManualTest, runWithdrawalAbortBankTest, runWithdrawalBankIntegratedTest, - runWallettestingTest, - runPaymentTest, ]; export interface TestRunSpec { @@ -166,7 +166,12 @@ export async function runTests(spec: TestRunSpec) { JSON.stringify({ testResults }, undefined, 2), ); console.log(`See ${resultsFile} for details`); + console.log(`Skipped: ${numSkip}/${numTotal}`); + console.log(`Failed: ${numFail}/${numTotal}`); console.log(`Passed: ${numPass}/${numTotal}`); + if (numPass < numTotal - numSkip) { + process.exit(1); + } } export function getTestInfo(): TestInfo[] { diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts index b13abac57..aed2ce5cb 100644 --- a/packages/taler-wallet-core/src/db.ts +++ b/packages/taler-wallet-core/src/db.ts @@ -1,5 +1,11 @@ -import { Stores } from "./types/dbTypes"; -import { openDatabase, Database, Store, Index } from "./util/query"; +import { MetaStores, Stores } from "./types/dbTypes"; +import { + openDatabase, + Database, + Store, + Index, + AnyStoreMap, +} from "./util/query"; import { IDBFactory, IDBDatabase, @@ -14,7 +20,11 @@ import { Logger } from "./util/logging"; * for all previous versions must be written, which should be * avoided. */ -const TALER_DB_NAME = "taler-wallet-prod-v1"; +const TALER_DB_NAME = "taler-wallet-main-v2"; + +const TALER_META_DB_NAME = "taler-wallet-meta"; + +const CURRENT_DB_CONFIG_KEY = "currentMainDbName"; /** * Current database minor version, should be incremented @@ -23,78 +33,134 @@ const TALER_DB_NAME = "taler-wallet-prod-v1"; * backwards-compatible way or object stores and indices * are added. */ -export const WALLET_DB_MINOR_VERSION = 3; +export const WALLET_DB_MINOR_VERSION = 1; const logger = new Logger("db.ts"); -/** - * Return a promise that resolves - * to the taler wallet db. - */ -export function openTalerDatabase( - idbFactory: IDBFactory, - onVersionChange: () => void, -): Promise { - const onUpgradeNeeded = ( - db: IDBDatabase, - oldVersion: number, - newVersion: number, - upgradeTransaction: IDBTransaction, - ): void => { - if (oldVersion === 0) { - for (const n in Stores) { - if ((Stores as any)[n] instanceof Store) { - const si: Store = (Stores as any)[n]; - const s = db.createObjectStore(si.name, si.storeParams); - for (const indexName in si as any) { - if ((si as any)[indexName] instanceof Index) { - const ii: Index = (si as any)[ - indexName - ]; - s.createIndex(ii.indexName, ii.keyPath, ii.options); - } +function upgradeFromStoreMap( + storeMap: AnyStoreMap, + db: IDBDatabase, + oldVersion: number, + newVersion: number, + upgradeTransaction: IDBTransaction, +): void { + if (oldVersion === 0) { + for (const n in storeMap) { + if ((storeMap as any)[n] instanceof Store) { + const si: Store = (storeMap as any)[n]; + const s = db.createObjectStore(si.name, si.storeParams); + for (const indexName in si as any) { + if ((si as any)[indexName] instanceof Index) { + const ii: Index = (si as any)[indexName]; + s.createIndex(ii.indexName, ii.keyPath, ii.options); } } } - return; - } - if (oldVersion === newVersion) { - return; } - logger.info(`upgrading database from ${oldVersion} to ${newVersion}`); - for (const n in Stores) { - if ((Stores as any)[n] instanceof Store) { - const si: Store = (Stores as any)[n]; - let s: IDBObjectStore; - const storeVersionAdded = si.storeParams?.versionAdded ?? 1; - if (storeVersionAdded > oldVersion) { - s = db.createObjectStore(si.name, si.storeParams); - } else { - s = upgradeTransaction.objectStore(si.name); - } - for (const indexName in si as any) { - if ((si as any)[indexName] instanceof Index) { - const ii: Index = (si as any)[indexName]; - const indexVersionAdded = ii.options?.versionAdded ?? 0; - if ( - indexVersionAdded > oldVersion || - storeVersionAdded > oldVersion - ) { - s.createIndex(ii.indexName, ii.keyPath, ii.options); - } + return; + } + if (oldVersion === newVersion) { + return; + } + logger.info(`upgrading database from ${oldVersion} to ${newVersion}`); + for (const n in Stores) { + if ((Stores as any)[n] instanceof Store) { + const si: Store = (Stores as any)[n]; + let s: IDBObjectStore; + const storeVersionAdded = si.storeParams?.versionAdded ?? 1; + if (storeVersionAdded > oldVersion) { + s = db.createObjectStore(si.name, si.storeParams); + } else { + s = upgradeTransaction.objectStore(si.name); + } + for (const indexName in si as any) { + if ((si as any)[indexName] instanceof Index) { + const ii: Index = (si as any)[indexName]; + const indexVersionAdded = ii.options?.versionAdded ?? 0; + if ( + indexVersionAdded > oldVersion || + storeVersionAdded > oldVersion + ) { + s.createIndex(ii.indexName, ii.keyPath, ii.options); } } } } - }; + } +} - return openDatabase( +function onTalerDbUpgradeNeeded( + db: IDBDatabase, + oldVersion: number, + newVersion: number, + upgradeTransaction: IDBTransaction, +) { + upgradeFromStoreMap(Stores, db, oldVersion, newVersion, upgradeTransaction); +} + +function onMetaDbUpgradeNeeded( + db: IDBDatabase, + oldVersion: number, + newVersion: number, + upgradeTransaction: IDBTransaction, +) { + upgradeFromStoreMap( + MetaStores, + db, + oldVersion, + newVersion, + upgradeTransaction, + ); +} + +/** + * Return a promise that resolves + * to the taler wallet db. + */ +export async function openTalerDatabase( + idbFactory: IDBFactory, + onVersionChange: () => void, +): Promise> { + const metaDbHandle = await openDatabase( + idbFactory, + TALER_META_DB_NAME, + 1, + () => {}, + onMetaDbUpgradeNeeded, + ); + + const metaDb = new Database(metaDbHandle, MetaStores); + let currentMainVersion: string | undefined; + await metaDb.runWithWriteTransaction([MetaStores.metaConfig], async (tx) => { + const dbVersionRecord = await tx.get( + MetaStores.metaConfig, + CURRENT_DB_CONFIG_KEY, + ); + if (!dbVersionRecord) { + currentMainVersion = TALER_DB_NAME; + await tx.put(MetaStores.metaConfig, { + key: CURRENT_DB_CONFIG_KEY, + value: TALER_DB_NAME, + }); + } else { + currentMainVersion = dbVersionRecord.key; + } + }); + + if (currentMainVersion !== TALER_DB_NAME) { + // In the future, the migration logic will be implemented here. + throw Error(`migration from database ${currentMainVersion} not supported`); + } + + const mainDbHandle = await openDatabase( idbFactory, TALER_DB_NAME, WALLET_DB_MINOR_VERSION, onVersionChange, - onUpgradeNeeded, + onTalerDbUpgradeNeeded, ); + + return new Database(mainDbHandle, Stores); } export function deleteTalerDatabase(idbFactory: IDBFactory): void { diff --git a/packages/taler-wallet-core/src/headless/helpers.ts b/packages/taler-wallet-core/src/headless/helpers.ts index 30b670032..3d380ad49 100644 --- a/packages/taler-wallet-core/src/headless/helpers.ts +++ b/packages/taler-wallet-core/src/headless/helpers.ts @@ -34,6 +34,7 @@ import { NodeHttpLib } from "./NodeHttpLib"; import { Logger } from "../util/logging"; import { SynchronousCryptoWorkerFactory } from "../crypto/workers/synchronousWorker"; import type { IDBFactory } from "idb-bridge/lib/idbtypes"; +import { Stores } from "../types/dbTypes"; const logger = new Logger("headless/helpers.ts"); @@ -149,9 +150,7 @@ export async function getDefaultNodeWallet( workerFactory = new SynchronousCryptoWorkerFactory(); } - const dbWrap = new Database(myDb); - - const w = new Wallet(dbWrap, myHttpLib, workerFactory); + const w = new Wallet(myDb, myHttpLib, workerFactory); if (args.notifyHandler) { w.addNotificationListener(args.notifyHandler); } diff --git a/packages/taler-wallet-core/src/index.ts b/packages/taler-wallet-core/src/index.ts index 3d52ed762..c446a0ffa 100644 --- a/packages/taler-wallet-core/src/index.ts +++ b/packages/taler-wallet-core/src/index.ts @@ -34,6 +34,7 @@ export { export * from "./operations/versions"; export * from "./db"; +export * from "./types/dbTypes"; // Internationalization export * from "./i18n"; diff --git a/packages/taler-wallet-core/src/operations/exchanges.ts b/packages/taler-wallet-core/src/operations/exchanges.ts index 52da6be62..7cc4fe101 100644 --- a/packages/taler-wallet-core/src/operations/exchanges.ts +++ b/packages/taler-wallet-core/src/operations/exchanges.ts @@ -29,6 +29,7 @@ import { DenominationStatus, WireFee, ExchangeUpdateReason, + MetaStores, } from "../types/dbTypes"; import { canonicalizeBaseUrl } from "../util/helpers"; import * as Amounts from "../util/amounts"; diff --git a/packages/taler-wallet-core/src/operations/state.ts b/packages/taler-wallet-core/src/operations/state.ts index 1733f13bb..60aee4c3f 100644 --- a/packages/taler-wallet-core/src/operations/state.ts +++ b/packages/taler-wallet-core/src/operations/state.ts @@ -23,6 +23,7 @@ import { PendingOperationsResponse } from "../types/pendingTypes"; import { WalletNotification } from "../types/notifications"; import { Database } from "../util/query"; import { openPromise, OpenedPromise } from "../util/promiseUtils"; +import { Stores } from "../types/dbTypes"; type NotificationListener = (n: WalletNotification) => void; @@ -59,7 +60,7 @@ export class InternalWalletState { // the actual value nullable. // Check if we are in a DB migration / garbage collection // and throw an error in that case. - public db: Database, + public db: Database, public http: HttpRequestLibrary, cryptoWorkerFactory: CryptoWorkerFactory, ) { diff --git a/packages/taler-wallet-core/src/types/dbTypes.ts b/packages/taler-wallet-core/src/types/dbTypes.ts index f55dcb2f6..0cfc8801b 100644 --- a/packages/taler-wallet-core/src/types/dbTypes.ts +++ b/packages/taler-wallet-core/src/types/dbTypes.ts @@ -1591,9 +1591,6 @@ class TipsStore extends Store<"tips", TipRecord> { this, "tipsByMerchantTipIdAndOriginIndex", ["merchantTipId", "merchantBaseUrl"], - { - versionAdded: 2, - }, ); } @@ -1657,7 +1654,7 @@ class BackupProvidersStore extends Store< BackupProviderRecord > { constructor() { - super("backupProviders", { keyPath: "baseUrl", versionAdded: 3 }); + super("backupProviders", { keyPath: "baseUrl" }); } } @@ -1688,3 +1685,13 @@ export const Stores = { bankWithdrawUris: new BankWithdrawUrisStore(), backupProviders: new BackupProvidersStore(), }; + +export class MetaConfigStore extends Store<"metaConfig", ConfigRecord> { + constructor() { + super("metaConfig", { keyPath: "key" }); + } +} + +export const MetaStores = { + metaConfig: new MetaConfigStore(), +}; diff --git a/packages/taler-wallet-core/src/util/query.ts b/packages/taler-wallet-core/src/util/query.ts index 35aab81e9..fdcab4fa1 100644 --- a/packages/taler-wallet-core/src/util/query.ts +++ b/packages/taler-wallet-core/src/util/query.ts @@ -269,6 +269,8 @@ class ResultStream { } } +export type AnyStoreMap = { [s: string]: Store }; + type StoreName = S extends Store ? N : never; type StoreContent = S extends Store ? R : never; type IndexRecord = Ind extends Index ? R : never; @@ -462,8 +464,7 @@ export class Index< } /** - * Return a promise that resolves - * to the taler wallet db. + * Return a promise that resolves to the opened IndexedDB database. */ export function openDatabase( idbFactory: IDBFactory, @@ -480,7 +481,7 @@ export function openDatabase( return new Promise((resolve, reject) => { const req = idbFactory.open(databaseName, databaseVersion); req.onerror = (e) => { - logger.error("taler database error", e); + logger.error("database error", e); reject(new Error("database error")); }; req.onsuccess = (e) => { @@ -508,8 +509,8 @@ export function openDatabase( }); } -export class Database { - constructor(private db: IDBDatabase) {} +export class Database { + constructor(private db: IDBDatabase, stores: StoreMap) {} static deleteDatabase(idbFactory: IDBFactory, dbName: string): void { idbFactory.deleteDatabase(dbName); @@ -571,10 +572,10 @@ export class Database { }); } - async get( - store: Store, + async get( + store: S, key: IDBValidKey, - ): Promise { + ): Promise | undefined> { const tx = this.db.transaction([store.name], "readonly"); const req = tx.objectStore(store.name).get(key); const v = await requestToPromise(req); @@ -634,14 +635,22 @@ export class Database { return new ResultStream>(req); } - async runWithReadTransaction>( + async runWithReadTransaction< + T, + N extends keyof StoreMap, + StoreTypes extends StoreMap[N] + >( stores: StoreTypes[], f: (t: TransactionHandle) => Promise, ): Promise { return runWithTransaction(this.db, stores, f, "readonly"); } - async runWithWriteTransaction>( + async runWithWriteTransaction< + T, + N extends keyof StoreMap, + StoreTypes extends StoreMap[N] + >( stores: StoreTypes[], f: (t: TransactionHandle) => Promise, ): Promise { diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts index 56e3d82d1..631ac9509 100644 --- a/packages/taler-wallet-core/src/wallet.ts +++ b/packages/taler-wallet-core/src/wallet.ts @@ -24,7 +24,7 @@ */ import { CryptoWorkerFactory } from "./crypto/workers/cryptoApi"; import { HttpRequestLibrary } from "./util/http"; -import { Database } from "./util/query"; +import { Database, Store } from "./util/query"; import { Amounts, AmountJson } from "./util/amounts"; @@ -52,6 +52,7 @@ import { ReserveRecordStatus, CoinSourceType, RefundState, + MetaStores, } from "./types/dbTypes"; import { CoinDumpJson, WithdrawUriInfoResponse } from "./types/talerTypes"; import { @@ -200,12 +201,12 @@ export class Wallet { private stopped = false; private memoRunRetryLoop = new AsyncOpMemoSingle(); - get db(): Database { + get db(): Database { return this.ws.db; } constructor( - db: Database, + db: Database, http: HttpRequestLibrary, cryptoWorkerFactory: CryptoWorkerFactory, ) { diff --git a/packages/taler-wallet-webextension/src/wxBackend.ts b/packages/taler-wallet-webextension/src/wxBackend.ts index e1dcdde49..95cd5f021 100644 --- a/packages/taler-wallet-webextension/src/wxBackend.ts +++ b/packages/taler-wallet-webextension/src/wxBackend.ts @@ -24,7 +24,6 @@ * Imports. */ import { isFirefox, getPermissionsApi } from "./compat"; -import MessageSender = chrome.runtime.MessageSender; import { extendedPermissions } from "./permissions"; import { @@ -40,6 +39,7 @@ import { CoreApiResponse, WalletDiagnostics, CoreApiResponseSuccess, + Stores, } from "taler-wallet-core"; import { BrowserHttpLib } from "./browserHttpLib"; import { BrowserCryptoWorkerFactory } from "./browserCryptoWorkerFactory"; @@ -50,7 +50,7 @@ import { BrowserCryptoWorkerFactory } from "./browserCryptoWorkerFactory"; */ let currentWallet: Wallet | undefined; -let currentDatabase: IDBDatabase | undefined; +let currentDatabase: Database | undefined; /** * Last version if an outdated DB, if applicable. @@ -135,7 +135,7 @@ async function dispatch( setupHeaderListener(); r = wrapResponse({ newValue: true }); } else { - await new Promise((resolve, reject) => { + await new Promise((resolve, reject) => { getPermissionsApi().remove(extendedPermissions, (rem) => { console.log("permissions removed:", rem); resolve(); @@ -246,7 +246,7 @@ async function reinitWallet(): Promise { const http = new BrowserHttpLib(); console.log("setting wallet"); const wallet = new Wallet( - new Database(currentDatabase), + currentDatabase, http, new BrowserCryptoWorkerFactory(), ); -- cgit v1.2.3