commit 3c05c86032e712ba02b3ad93d7b6d59d73f0b8c9
parent fa7459c710ba8bc13bc1fa15469f104fdd5fad86
Author: Florian Dold <dold@taler.net>
Date: Fri, 28 Aug 2026 23:14:35 +0200
wallet-core: convert database dumps across backends
Diffstat:
5 files changed, 396 insertions(+), 3 deletions(-)
diff --git a/packages/taler-util/src/notifications.ts b/packages/taler-util/src/notifications.ts
@@ -386,7 +386,10 @@ export interface IdleNotification {
/** Progress while startup fixups or a database migration hold the DB gate. */
export interface DatabaseMaintenanceProgressNotification {
type: NotificationType.DatabaseMaintenanceProgress;
- operation: "indexeddb-fixup" | "indexeddb-to-native-migration";
+ operation:
+ | "indexeddb-fixup"
+ | "indexeddb-to-native-migration"
+ | "cross-backend-import";
/** Token of the API request that initiated this operation, when available. */
progressToken?: string;
phase: "fixup" | "copy" | "verify" | "complete" | "failed";
diff --git a/packages/taler-wallet-core/src/db/handle.ts b/packages/taler-wallet-core/src/db/handle.ts
@@ -46,6 +46,8 @@ export interface WalletDbMigrationOptions {
progressToken?: string;
}
+export type WalletDbBackend = "indexeddb" | "sqlite";
+
/**
* Wallet-level work that must become visible in the same atomic import as the
* restored records, such as rebuilding materialized views and counters.
diff --git a/packages/taler-wallet-core/src/db/migration/converter.ts b/packages/taler-wallet-core/src/db/migration/converter.ts
@@ -528,6 +528,9 @@ export interface DbConversionOptions {
/** API progress token to attach to maintenance notifications. */
progressToken?: string;
+ /** Maintenance operation reported by this use of the converter. */
+ operation?: DatabaseMaintenanceProgressNotification["operation"];
+
/**
* Called after a progress notification has been delivered to the source
* handle. Throwing aborts the conversion, which lets callers inject a
@@ -667,13 +670,14 @@ export async function convertWalletDb(
options: DbConversionOptions = {},
): Promise<DbConversionReport> {
const copied: Record<string, number> = {};
+ const operation = options.operation ?? "indexeddb-to-native-migration";
// Let the host display maintenance immediately. Counting is cheap, but an
// old emulated database can still spend noticeable time opening its first
// transaction on a phone.
const initialNotification: DatabaseMaintenanceProgressNotification = {
type: NotificationType.DatabaseMaintenanceProgress,
- operation: "indexeddb-to-native-migration",
+ operation,
phase: "copy",
...(options.progressToken ? { progressToken: options.progressToken } : {}),
completedSteps: 0,
@@ -718,7 +722,7 @@ export async function convertWalletDb(
const phasePercent = Math.floor((50 * completedUnits) / totalUnits);
const notification: DatabaseMaintenanceProgressNotification = {
type: NotificationType.DatabaseMaintenanceProgress,
- operation: "indexeddb-to-native-migration",
+ operation,
phase,
...(options.progressToken
? { progressToken: options.progressToken }
diff --git a/packages/taler-wallet-core/src/db/migration/import.test.ts b/packages/taler-wallet-core/src/db/migration/import.test.ts
@@ -0,0 +1,216 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import assert from "node:assert";
+import { test } from "node:test";
+
+import { createNodeHelperSqlite3Impl } from "@gnu-taler/idb-bridge/node-helper-sqlite3-impl";
+import {
+ CancellationToken,
+ DatabaseMaintenanceProgressNotification,
+ NotificationType,
+} from "@gnu-taler/taler-util";
+
+import { ConfigRecordKey } from "../records.js";
+import { makeIdbRunner, makeSqliteRunner } from "../testing/runners.js";
+import { getWalletDbDumpBackend, importWalletDbDump } from "./import.js";
+
+test("cross import recognizes only one current dump format", () => {
+ assert.strictEqual(getWalletDbDumpBackend({ databases: {} }), "indexeddb");
+ assert.strictEqual(
+ getWalletDbDumpBackend({ schemaVersion: 1, tables: {} }),
+ "sqlite",
+ );
+ assert.strictEqual(getWalletDbDumpBackend({}), undefined);
+ assert.strictEqual(
+ getWalletDbDumpBackend({
+ databases: {},
+ schemaVersion: 1,
+ tables: {},
+ }),
+ undefined,
+ );
+});
+
+for (const direction of [
+ ["indexeddb", makeIdbRunner, "sqlite", makeSqliteRunner],
+ ["sqlite", makeSqliteRunner, "indexeddb", makeIdbRunner],
+] as const) {
+ const [sourceName, makeSource, targetName, makeTarget] = direction;
+ test(`cross import converts ${sourceName} to ${targetName}`, async () => {
+ const source = await makeSource();
+ const target = await makeTarget();
+ const sqlite3Impl = await createNodeHelperSqlite3Impl({
+ enableTracing: false,
+ });
+ const notifications: DatabaseMaintenanceProgressNotification[] = [];
+ target.setNotificationSink((notification) => {
+ if (notification.type === NotificationType.DatabaseMaintenanceProgress) {
+ notifications.push(notification);
+ }
+ });
+ try {
+ await source.runReadWriteTx(async (tx) => {
+ await tx.upsertConfig({
+ key: ConfigRecordKey.TestLoopTx,
+ value: 2,
+ });
+ await tx.upsertTombstone({ id: "source-tombstone" });
+ });
+ await target.runReadWriteTx(async (tx) => {
+ await tx.upsertConfig({
+ key: ConfigRecordKey.TestLoopTx,
+ value: 1,
+ });
+ await tx.upsertTombstone({ id: "old-target-tombstone" });
+ });
+
+ await importWalletDbDump(
+ sqlite3Impl,
+ target,
+ await source.exportDatabase(),
+ async (tx) => {
+ await tx.upsertTombstone({ id: "import-finalizer" });
+ },
+ { progressToken: "cross-import-test" },
+ );
+
+ await target.runReadWriteTx(async (tx) => {
+ assert.strictEqual(
+ (await tx.getConfig(ConfigRecordKey.TestLoopTx))?.value,
+ 2,
+ );
+ const ids = (await tx.listAllTombstones()).map((x) => x.id).sort();
+ assert.deepStrictEqual(ids, ["import-finalizer", "source-tombstone"]);
+ });
+ assert.ok(
+ notifications.some(
+ (n) =>
+ n.operation === "cross-backend-import" &&
+ n.phase === "complete" &&
+ n.progressToken === "cross-import-test" &&
+ n.completionPercent === 100,
+ ),
+ );
+ } finally {
+ await source.close();
+ await target.close();
+ }
+ });
+}
+
+test("cross import leaves the target authoritative when finalization fails", async () => {
+ const source = await makeIdbRunner();
+ const target = await makeSqliteRunner();
+ const sqlite3Impl = await createNodeHelperSqlite3Impl({
+ enableTracing: false,
+ });
+ try {
+ await source.runReadWriteTx((tx) =>
+ tx.upsertConfig({
+ key: ConfigRecordKey.TestLoopTx,
+ value: 2,
+ }),
+ );
+ await target.runReadWriteTx((tx) =>
+ tx.upsertConfig({
+ key: ConfigRecordKey.TestLoopTx,
+ value: 1,
+ }),
+ );
+
+ await assert.rejects(
+ importWalletDbDump(
+ sqlite3Impl,
+ target,
+ await source.exportDatabase(),
+ async (tx) => {
+ await tx.upsertConfig({
+ key: ConfigRecordKey.TestLoopTx,
+ value: 3,
+ });
+ throw Error("injected finalizer failure");
+ },
+ ),
+ /injected finalizer failure/,
+ );
+ assert.strictEqual(
+ (
+ await target.runReadWriteTx((tx) =>
+ tx.getConfig(ConfigRecordKey.TestLoopTx),
+ )
+ )?.value,
+ 1,
+ );
+ } finally {
+ await source.close();
+ await target.close();
+ }
+});
+
+test("cross import cancellation does not publish a partial target", async () => {
+ const source = await makeIdbRunner();
+ const target = await makeSqliteRunner();
+ const sqlite3Impl = await createNodeHelperSqlite3Impl({
+ enableTracing: false,
+ });
+ const cts = CancellationToken.create();
+ target.setNotificationSink((notification) => {
+ if (
+ notification.type === NotificationType.DatabaseMaintenanceProgress &&
+ notification.operation === "cross-backend-import" &&
+ notification.phase === "copy" &&
+ (notification.processedRecords ?? 0) >= 100
+ ) {
+ cts.cancel();
+ }
+ });
+ try {
+ await source.runReadWriteTx(async (tx) => {
+ for (let i = 0; i < 250; i++) {
+ await tx.upsertTombstone({ id: `cancel-source-${i}` });
+ }
+ });
+ await target.runReadWriteTx((tx) =>
+ tx.upsertConfig({
+ key: ConfigRecordKey.TestLoopTx,
+ value: 1,
+ }),
+ );
+
+ await assert.rejects(
+ importWalletDbDump(
+ sqlite3Impl,
+ target,
+ await source.exportDatabase(),
+ async () => {},
+ { cancellationToken: cts.token },
+ ),
+ CancellationToken.CancellationError,
+ );
+ assert.strictEqual(
+ (
+ await target.runReadWriteTx((tx) =>
+ tx.getConfig(ConfigRecordKey.TestLoopTx),
+ )
+ )?.value,
+ 1,
+ );
+ } finally {
+ await source.close();
+ await target.close();
+ }
+});
diff --git a/packages/taler-wallet-core/src/db/migration/import.ts b/packages/taler-wallet-core/src/db/migration/import.ts
@@ -0,0 +1,168 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+/**
+ * Restore a dump produced by either wallet database backend.
+ *
+ * A foreign dump is first materialized through its own backend in a private
+ * in-memory SQLite database. Both schemas coexist in that one connection,
+ * allowing the regular DAL converter to copy and verify the wallet without a
+ * temporary file. The converted destination is then exported in the live
+ * backend's format and handed to its normal crash-atomic importer.
+ */
+
+import {
+ BridgeIDBFactory,
+ createSqliteBackendOverDb,
+ Sqlite3Database,
+ Sqlite3Interface,
+} from "@gnu-taler/idb-bridge";
+import {
+ CancellationToken,
+ DatabaseMaintenanceProgressNotification,
+ getErrorDetailFromException,
+ NotificationType,
+} from "@gnu-taler/taler-util";
+
+import {
+ WalletDbBackend,
+ WalletDbHandle,
+ WalletDbImportFinalizer,
+ WalletDbMigrationOptions,
+} from "../handle.js";
+import { IdbWalletDbHandle } from "../indexeddb/handle.js";
+import { openNativeSqliteWalletDb } from "../sqlite/database.js";
+import { SqliteWalletDbHandle } from "../sqlite/handle.js";
+import { convertWalletDb, DB_CONVERSION_STEP_COUNT } from "./converter.js";
+
+export function getWalletDbDumpBackend(
+ dump: unknown,
+): WalletDbBackend | undefined {
+ if (dump === null || typeof dump !== "object" || Array.isArray(dump)) {
+ return undefined;
+ }
+ const obj = dump as Record<string, unknown>;
+ const isIndexedDb = Object.hasOwn(obj, "databases");
+ const isSqlite =
+ Object.hasOwn(obj, "schemaVersion") && Object.hasOwn(obj, "tables");
+ if (isIndexedDb === isSqlite) {
+ return undefined;
+ }
+ return isIndexedDb ? "indexeddb" : "sqlite";
+}
+
+export async function importWalletDbDump(
+ sqlite3Impl: Sqlite3Interface,
+ target: WalletDbHandle,
+ dump: unknown,
+ finalize: WalletDbImportFinalizer,
+ options: WalletDbMigrationOptions = {},
+): Promise<void> {
+ const sourceBackend = getWalletDbDumpBackend(dump);
+ if (!sourceBackend) {
+ throw Error("doesn't look like a valid wallet database dump");
+ }
+ if (sourceBackend === target.name) {
+ options.cancellationToken?.throwIfCancelled();
+ await target.importDatabase(dump, finalize);
+ return;
+ }
+ if (target.name !== "indexeddb" && target.name !== "sqlite") {
+ throw Error(`unknown wallet database backend: ${target.name}`);
+ }
+
+ const operation = "cross-backend-import" as const;
+ let lastProgress: DatabaseMaintenanceProgressNotification | undefined;
+ let idbHandle: IdbWalletDbHandle | undefined;
+ let sqliteHandle: SqliteWalletDbHandle | undefined;
+ let rawDb: Sqlite3Database | undefined;
+ try {
+ rawDb = await sqlite3Impl.open(":memory:");
+ const idbBackend = await createSqliteBackendOverDb(sqlite3Impl, rawDb);
+ idbHandle = new IdbWalletDbHandle(new BridgeIDBFactory(idbBackend));
+ sqliteHandle = new SqliteWalletDbHandle(
+ await openNativeSqliteWalletDb(rawDb),
+ );
+
+ const source = sourceBackend === "indexeddb" ? idbHandle : sqliteHandle;
+ const destination = target.name === "indexeddb" ? idbHandle : sqliteHandle;
+ source.setNotificationSink((notification) =>
+ target.emitNotification(notification),
+ );
+
+ options.cancellationToken?.throwIfCancelled();
+ await source.importDatabase(dump, async () => {});
+ options.cancellationToken?.throwIfCancelled();
+
+ const report = await convertWalletDb(source, destination, {
+ cancellationToken:
+ options.cancellationToken ?? CancellationToken.CONTINUE,
+ progressToken: options.progressToken,
+ operation,
+ onProgress(notification) {
+ lastProgress = notification;
+ },
+ });
+ options.cancellationToken?.throwIfCancelled();
+
+ const convertedDump = await destination.exportDatabase();
+ options.cancellationToken?.throwIfCancelled();
+ await target.importDatabase(convertedDump, finalize);
+
+ target.emitNotification({
+ type: NotificationType.DatabaseMaintenanceProgress,
+ operation,
+ phase: "complete",
+ ...(options.progressToken
+ ? { progressToken: options.progressToken }
+ : {}),
+ completedSteps: DB_CONVERSION_STEP_COUNT,
+ totalSteps: DB_CONVERSION_STEP_COUNT,
+ processedRecords: report.totalRecords,
+ totalRecords: report.totalRecords,
+ completionPercent: 100,
+ });
+ } catch (e) {
+ target.emitNotification({
+ type: NotificationType.DatabaseMaintenanceProgress,
+ operation,
+ phase: "failed",
+ ...(options.progressToken
+ ? { progressToken: options.progressToken }
+ : {}),
+ completedSteps: lastProgress?.completedSteps ?? 0,
+ totalSteps: DB_CONVERSION_STEP_COUNT,
+ ...(lastProgress?.processedRecords !== undefined
+ ? { processedRecords: lastProgress.processedRecords }
+ : {}),
+ ...(lastProgress?.totalRecords !== undefined
+ ? { totalRecords: lastProgress.totalRecords }
+ : {}),
+ completionPercent: lastProgress?.completionPercent ?? 0,
+ error: getErrorDetailFromException(e),
+ });
+ throw e;
+ } finally {
+ if (idbHandle) {
+ await idbHandle.close().catch(() => {});
+ }
+ if (sqliteHandle) {
+ await sqliteHandle.close().catch(() => {});
+ } else if (rawDb) {
+ await rawDb.close().catch(() => {});
+ }
+ }
+}
