commit 15f2856f1fab6bc115de155c9a97a512ae856f37
parent 56995ed270383bf3d7b1f4a1b7998d8fe809e734
Author: Florian Dold <dold@taler.net>
Date: Thu, 6 Aug 2026 20:33:10 +0200
wallet: migrate the wallet database to the native schema in place
The two schemas' table names are disjoint, so the copy happens inside the file
the wallet already has, which is the only kind of migration the mobile wallets
can run. Behind features.migrateNativeDb, off by default; the emulation's
tables are renamed aside rather than dropped, and kept for a month.
Issue: https://bugs.taler.net/n/11718
Diffstat:
13 files changed, 1037 insertions(+), 30 deletions(-)
diff --git a/packages/taler-util/src/types-taler-wallet.ts b/packages/taler-util/src/types-taler-wallet.ts
@@ -469,6 +469,21 @@ export interface WalletRunConfig {
*/
features: {
allowHttp: boolean;
+
+ /**
+ * Migrate the wallet database to wallet-core's native sqlite schema,
+ * replacing the IndexedDB emulation it has been stored in so far.
+ *
+ * Checked whenever the wallet is initialized, so a client that offers
+ * this as a setting only has to initialize the wallet again for it to
+ * take effect. Off by default: the native schema is still experimental.
+ *
+ * The migration happens once and is not undone by clearing the flag --
+ * afterwards the database *is* a native one. It also only ever runs
+ * where the host stores the wallet in a sqlite file, which excludes the
+ * browser extension.
+ */
+ migrateNativeDb: boolean;
};
/**
diff --git a/packages/taler-wallet-cli/src/index.ts b/packages/taler-wallet-cli/src/index.ts
@@ -86,7 +86,9 @@ import { JsonMessage, runRpcServer } from "@gnu-taler/taler-util/twrpc";
import {
convertWalletDbFile,
createNativeWalletHost2,
+ inspectWalletDbPath,
nativeCrypto,
+ rollbackWalletDbMigration,
Wallet,
WalletApiOperation,
WalletCoreApiClient,
@@ -2956,6 +2958,38 @@ advancedCli
});
advancedCli
+ .subcommand("dbMigrationInfo", "db-migration-info", {
+ help: "Show which schema a wallet database file uses.",
+ mark: "experimental",
+ })
+ .requiredArgument("dbfile", clk.STRING, {
+ help: "Wallet database file to inspect.",
+ })
+ .action(async (args) => {
+ const info = await inspectWalletDbPath(args.dbMigrationInfo.dbfile);
+ console.log(j2s(info));
+ });
+
+advancedCli
+ .subcommand("dbMigrationRollback", "db-migration-rollback", {
+ help: "Undo an in-place migration to the native database schema.",
+ mark: "experimental",
+ })
+ .requiredArgument("dbfile", clk.STRING, {
+ help: "Wallet database file to roll back.",
+ })
+ .action(async (args) => {
+ // Everything the wallet did since the migration lives in the native
+ // tables and is not carried back, so say so rather than report success
+ // and let it be discovered later.
+ await rollbackWalletDbMigration(args.dbMigrationRollback.dbfile);
+ console.log(
+ `rolled back to the database as it was before the migration;` +
+ ` anything the wallet did since then is not part of it.`,
+ );
+ });
+
+advancedCli
.subcommand("diagnostics", "diagnostics", {
help: "Print diagnostics info.",
})
diff --git a/packages/taler-wallet-core/src/db-native-migration.test.ts b/packages/taler-wallet-core/src/db-native-migration.test.ts
@@ -0,0 +1,263 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+/**
+ * Tests for the in-place migration to the native schema.
+ *
+ * The source database is populated by running the whole conformance corpus
+ * against it, so the migration faces every record type the suite can produce.
+ * The copy itself is verified record by record by the converter underneath;
+ * what these cases are about is the part that only in-place migration has --
+ * which schema a file is opened with afterwards, what happens to the tables
+ * that were migrated away from, and what an interrupted attempt leaves behind.
+ */
+
+import assert from "node:assert";
+import { test } from "node:test";
+
+import {
+ BridgeIDBFactory,
+ createSqliteBackendOverDb,
+ Sqlite3Database,
+} from "@gnu-taler/idb-bridge";
+import { createNodeHelperSqlite3Impl } from "@gnu-taler/idb-bridge/node-helper-sqlite3-impl";
+
+import {
+ dropExpiredMigrationBackup,
+ inspectWalletDbFile,
+ migrateWalletDbToNative,
+ readNativeMigrationInfo,
+ restoreMigrationBackup,
+} from "./db-native-migration.js";
+import { IDB_BACKUP_PREFIX, IDB_EMULATION_TABLES } from "./db-sqlite-schema.js";
+import { IdbWalletDbHandle } from "./dbtx-handle-impl.js";
+import { conformanceCases } from "./dbtx-conformance-cases.js";
+import { ConformanceAsserts } from "./dbtx-conformance.js";
+
+/** Assertions that ignore case-internal failures: only the data matters. */
+const quietAsserts: ConformanceAsserts = {
+ equal: () => {},
+ deepEqual: () => {},
+ ok: () => {},
+ fail: () => {
+ throw Error("unreachable");
+ },
+};
+
+async function listTables(db: Sqlite3Database): Promise<string[]> {
+ const rows = await (
+ await db.prepare(
+ "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name",
+ )
+ ).getAll();
+ return rows.map((r) => String(r.name));
+}
+
+async function countRows(db: Sqlite3Database, table: string): Promise<number> {
+ const row = await (
+ await db.prepare(`SELECT COUNT(*) AS n FROM "${table}"`)
+ ).getFirst({});
+ return Number(row?.n);
+}
+
+/**
+ * An emulation-backed wallet database with the conformance corpus in it, over
+ * a connection the caller keeps: the migration needs that same connection.
+ */
+async function makePopulatedIdbDb(): Promise<{
+ db: Sqlite3Database;
+ handle: IdbWalletDbHandle;
+}> {
+ const imp = await createNodeHelperSqlite3Impl({ enableTracing: false });
+ const db = await imp.open(":memory:");
+ const backend = await createSqliteBackendOverDb(imp, db);
+ BridgeIDBFactory.enableTracing = false;
+ const handle = new IdbWalletDbHandle(new BridgeIDBFactory(backend));
+ await handle.ensureOpen();
+ for (const c of conformanceCases) {
+ try {
+ await c.run(quietAsserts, handle as any);
+ } catch (e) {
+ // A case failing its own assertions is the conformance suite's concern;
+ // what matters here is whatever data it managed to write.
+ }
+ }
+ return { db, handle };
+}
+
+test("native migration: happens in the same file and switches it over", async () => {
+ const { db, handle } = await makePopulatedIdbDb();
+
+ assert.strictEqual(await inspectWalletDbFile(db), "indexeddb");
+
+ const {
+ handle: native,
+ report,
+ info,
+ } = await migrateWalletDbToNative(db, handle);
+
+ assert.ok(
+ report.totalRecords >= 100,
+ `only ${report.totalRecords} records migrated -- the corpus did not` +
+ ` populate the source, so the migration proved nothing`,
+ );
+ assert.strictEqual(info.status, "complete");
+ assert.strictEqual(info.recordsCopied, report.totalRecords);
+ assert.strictEqual(info.backupStatus, "retained");
+ assert.ok(info.backupExpiresAt! > info.finishedAt!);
+
+ // The file now opens natively, without being told to.
+ assert.strictEqual(await inspectWalletDbFile(db), "native");
+
+ const tables = await listTables(db);
+ for (const t of IDB_EMULATION_TABLES) {
+ assert.ok(
+ !tables.includes(t),
+ `${t} is still there, so the emulation would keep being used`,
+ );
+ assert.ok(
+ tables.includes(`${IDB_BACKUP_PREFIX}${t}`),
+ `${t} was not retained as a backup`,
+ );
+ }
+ // The retained copy still holds the records it held before.
+ assert.ok((await countRows(db, `${IDB_BACKUP_PREFIX}object_data`)) > 0);
+
+ // The migrated database is usable through the handle the wallet gets.
+ const coins = await native.runReadWriteTx((tx) => tx.listAllCoins());
+ assert.ok(coins.length > 0, "no coins survived the migration");
+
+ await native.close();
+});
+
+test("native migration: clearing the wallet leaves the retained backup", async () => {
+ const { db, handle } = await makePopulatedIdbDb();
+ const { handle: native } = await migrateWalletDbToNative(db, handle);
+
+ const before = await countRows(db, `${IDB_BACKUP_PREFIX}object_data`);
+ assert.ok(before > 0);
+
+ // clearDatabase enumerates the tables in the file; the emulation's retained
+ // tables are in that same file and are not the wallet's data.
+ await native.clearDatabase();
+
+ assert.strictEqual(
+ await countRows(db, `${IDB_BACKUP_PREFIX}object_data`),
+ before,
+ );
+ await native.close();
+});
+
+test("native migration: the backup is dropped only once it expires", async () => {
+ const { db, handle } = await makePopulatedIdbDb();
+ const { handle: native, info } = await migrateWalletDbToNative(db, handle);
+
+ assert.strictEqual(
+ await dropExpiredMigrationBackup(db, info.backupExpiresAt! - 1),
+ false,
+ "the backup went away before its retention was over",
+ );
+ assert.ok((await listTables(db)).includes(`${IDB_BACKUP_PREFIX}object_data`));
+
+ assert.strictEqual(
+ await dropExpiredMigrationBackup(db, info.backupExpiresAt!),
+ true,
+ );
+ const tables = await listTables(db);
+ for (const t of IDB_EMULATION_TABLES) {
+ assert.ok(!tables.includes(`${IDB_BACKUP_PREFIX}${t}`));
+ }
+ assert.strictEqual(
+ (await readNativeMigrationInfo(db))?.backupStatus,
+ "dropped",
+ );
+ // Dropping the backup does not change which schema the file is read with.
+ assert.strictEqual(await inspectWalletDbFile(db), "native");
+
+ // And a second call has nothing left to do.
+ assert.strictEqual(
+ await dropExpiredMigrationBackup(db, info.backupExpiresAt!),
+ false,
+ );
+ await native.close();
+});
+
+test("native migration: the retained backup can be put back", async () => {
+ const { db, handle } = await makePopulatedIdbDb();
+ const rowsBefore = await countRows(db, "object_data");
+ await migrateWalletDbToNative(db, handle);
+
+ // Not closing the handle first: closing it closes the connection this test
+ // still holds, and the file is what the restore works on. In production
+ // the restore runs against a wallet that is not running at all.
+ await restoreMigrationBackup(db);
+
+ assert.strictEqual(await inspectWalletDbFile(db), "indexeddb");
+ assert.strictEqual(await countRows(db, "object_data"), rowsBefore);
+ const info = await readNativeMigrationInfo(db);
+ assert.strictEqual(info?.status, "rolled-back");
+ assert.strictEqual(info?.backupStatus, "restored");
+
+ // A rolled-back database is not migrated again behind the user's back.
+ await assert.rejects(
+ () => migrateWalletDbToNative(db, handle),
+ /rolled back/,
+ );
+});
+
+test("native migration: an interrupted attempt keeps the old database", async () => {
+ const { db, handle } = await makePopulatedIdbDb();
+
+ // What an interruption leaves behind: the native tables exist and hold a
+ // partial copy, the bookkeeping says an attempt was under way, and the
+ // emulation's tables are untouched because the renames never ran.
+ await migrateWalletDbToNative(db, handle);
+ // Undo the completion so the file looks interrupted rather than migrated:
+ // the bookkeeping says an attempt was under way and the emulation's tables
+ // are back where a migration that never reached its renames left them.
+ await (
+ await db.prepare(
+ "UPDATE idb_migration SET status = 'running', finished_at = NULL," +
+ " backup_status = NULL, backup_expires_at = NULL WHERE id = 1",
+ )
+ ).run({});
+ for (const t of IDB_EMULATION_TABLES) {
+ await (
+ await db.prepare(
+ `ALTER TABLE "${IDB_BACKUP_PREFIX}${t}" RENAME TO "${t}"`,
+ )
+ ).run({});
+ }
+
+ assert.strictEqual(
+ await inspectWalletDbFile(db),
+ "indexeddb",
+ "an interrupted migration must leave the emulation authoritative",
+ );
+
+ // Retrying discards the partial copy rather than adding to it.
+ const { handle: native, report } = await migrateWalletDbToNative(db, handle);
+ const coins = await native.runReadWriteTx((tx) => tx.listAllCoins());
+ const seen = new Set(coins.map((c) => c.coinPub));
+ assert.strictEqual(
+ seen.size,
+ coins.length,
+ "the retry duplicated records from the interrupted attempt",
+ );
+ assert.ok(report.totalRecords >= 100);
+ assert.strictEqual(await inspectWalletDbFile(db), "native");
+ await native.close();
+});
diff --git a/packages/taler-wallet-core/src/db-native-migration.ts b/packages/taler-wallet-core/src/db-native-migration.ts
@@ -0,0 +1,410 @@
+/*
+ This file is part of GNU Taler
+ (C) 2026 Taler Systems S.A.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+/**
+ * In-place migration of a wallet database from the IndexedDB emulation to the
+ * native schema.
+ *
+ * Both schemas are sqlite tables and their names do not overlap, so the
+ * migration happens inside the one file the wallet already has: nothing is
+ * written next to it and nothing is swapped afterwards. That matters because
+ * the platforms this migration exists for -- the mobile wallets -- hand
+ * wallet-core a database and no filesystem to put a second one in.
+ *
+ * The order of operations is what makes an interrupted migration safe. The
+ * emulation's tables are read-only throughout and are renamed out of the way
+ * only after the copy has been verified, in the same transaction that records
+ * the migration as complete. So at every instant exactly one of the two
+ * schemas is the authoritative copy, and which one it is can be read back
+ * from the file:
+ *
+ * - no idb_migration row: the emulation's tables are the wallet.
+ * - status 'running': an attempt was interrupted. The emulation's tables are
+ * still the wallet; the native tables hold a partial copy and are discarded
+ * when the migration is retried.
+ * - status 'complete': the native tables are the wallet. The emulation's
+ * tables are still in the file under their idb_backup_ names.
+ *
+ * The backup is kept for {@link MIGRATION_BACKUP_RETENTION} rather than
+ * dropped at the end: a migration that copies every record and verifies it can
+ * still turn out to have produced a wallet that misbehaves for a reason nobody
+ * anticipated, and until that window closes the original is one statement
+ * away. {@link restoreMigrationBackup} is that statement.
+ */
+
+import { Duration, Logger } from "@gnu-taler/taler-util";
+import type { Sqlite3Database } from "@gnu-taler/idb-bridge";
+
+import { convertWalletDb, DbConversionReport } from "./db-converter.js";
+import { IDB_BACKUP_PREFIX, IDB_EMULATION_TABLES } from "./db-sqlite-schema.js";
+import { SqliteWalletDbHandle } from "./dbtx-handle-impl.js";
+import { WalletDbHandle } from "./dbtx-handle.js";
+import {
+ clearNativeSqliteWalletDb,
+ openNativeSqliteWalletDb,
+ SqliteTxControl,
+} from "./dbtx-sqlite.js";
+
+const logger = new Logger("db-native-migration.ts");
+
+/**
+ * How long the renamed emulation tables are kept after a successful
+ * migration.
+ *
+ * Long enough that a wallet used every few days gets several chances to
+ * expose a problem before the original goes away, short enough that a
+ * wallet's storage does not carry two copies of itself indefinitely.
+ */
+export const MIGRATION_BACKUP_RETENTION = Duration.fromSpec({ months: 1 });
+
+/** The retention as the microseconds the schema's timestamps are in. */
+function retentionMicros(): number {
+ const ms = MIGRATION_BACKUP_RETENTION.d_ms;
+ // fromSpec never yields "forever", but narrowing rather than casting means
+ // a retention that later becomes configurable cannot silently overflow into
+ // a negative expiry. MAX_SAFE_INTEGER is this schema's "never".
+ return ms === "forever" ? Number.MAX_SAFE_INTEGER : ms * 1000;
+}
+
+/** Which schema the records in a wallet database file are stored in. */
+export type WalletDbFileKind = "empty" | "indexeddb" | "native";
+
+/**
+ * 'rolled-back' is terminal: the emulation tables were put back by
+ * {@link restoreMigrationBackup}, and the wallet does not migrate again on its
+ * own, because whoever rolled back did so to stop using the native schema.
+ */
+export type MigrationStatus = "running" | "complete" | "rolled-back";
+
+export type MigrationBackupStatus = "retained" | "dropped" | "restored";
+
+export interface NativeMigrationInfo {
+ status: MigrationStatus;
+ /** Microseconds since the epoch, as everywhere in the native schema. */
+ startedAt: number;
+ finishedAt?: number;
+ recordsCopied?: number;
+ backupStatus?: MigrationBackupStatus;
+ backupExpiresAt?: number;
+}
+
+/** Current time in the microseconds the native schema's timestamps use. */
+function nowMicros(): number {
+ return Date.now() * 1000;
+}
+
+function backupTableName(table: string): string {
+ return `${IDB_BACKUP_PREFIX}${table}`;
+}
+
+async function tableExists(
+ db: Sqlite3Database,
+ name: string,
+): Promise<boolean> {
+ const row = await (
+ await db.prepare(
+ "SELECT name FROM sqlite_master WHERE type = 'table' AND name = $name",
+ )
+ ).getFirst({ name });
+ return row != null;
+}
+
+/**
+ * Read the migration bookkeeping, if this file has any.
+ *
+ * Tolerates a file that has never seen the native schema: the table itself is
+ * absent there, which is not an error but the most common case.
+ */
+export async function readNativeMigrationInfo(
+ db: Sqlite3Database,
+): Promise<NativeMigrationInfo | undefined> {
+ if (!(await tableExists(db, "idb_migration"))) {
+ return undefined;
+ }
+ const row = await (
+ await db.prepare("SELECT * FROM idb_migration WHERE id = 1")
+ ).getFirst({});
+ if (!row) {
+ return undefined;
+ }
+ const optNum = (v: unknown): number | undefined =>
+ v == null ? undefined : Number(v);
+ return {
+ status: String(row.status) as MigrationStatus,
+ startedAt: Number(row.started_at),
+ finishedAt: optNum(row.finished_at),
+ recordsCopied: optNum(row.records_copied),
+ backupStatus: (row.backup_status ?? undefined) as
+ | MigrationBackupStatus
+ | undefined,
+ backupExpiresAt: optNum(row.backup_expires_at),
+ };
+}
+
+/**
+ * Decide which schema holds the wallet's records in an open database file.
+ *
+ * The host has to ask before it opens either backend over the file, because
+ * both create their tables with IF NOT EXISTS: opening the wrong one does not
+ * fail, it produces an empty wallet.
+ */
+export async function inspectWalletDbFile(
+ db: Sqlite3Database,
+): Promise<WalletDbFileKind> {
+ const info = await readNativeMigrationInfo(db);
+ if (info?.status === "complete") {
+ return "native";
+ }
+ // Before the renames, the emulation's tables are authoritative even when a
+ // partial native copy exists next to them.
+ if (await tableExists(db, "object_data")) {
+ return "indexeddb";
+ }
+ if (await tableExists(db, "schema_migrations")) {
+ return "native";
+ }
+ return "empty";
+}
+
+/**
+ * Run f inside one native sqlite transaction on db.
+ *
+ * The migration owns the connection while it runs, so it does not go through
+ * the wallet's transaction queue; it does need the same explicit
+ * BEGIN/COMMIT, since exec() would commit implicitly between statements.
+ */
+async function inTransaction(
+ txc: SqliteTxControl,
+ f: () => Promise<void>,
+): Promise<void> {
+ await txc.begin();
+ try {
+ await f();
+ await txc.commit();
+ } catch (e) {
+ try {
+ await txc.rollback();
+ } catch (rollbackErr) {
+ logger.warn(`rollback failed: ${rollbackErr}`);
+ }
+ throw e;
+ }
+}
+
+export interface NativeMigrationResult {
+ handle: SqliteWalletDbHandle;
+ report: DbConversionReport;
+ info: NativeMigrationInfo;
+}
+
+/**
+ * Migrate the wallet records in db from the emulation to the native schema.
+ *
+ * `src` must be the open IndexedDB-emulation handle over the same connection:
+ * opening it is what replays the fixup log, so the records this copies are
+ * already repaired -- the native schema has no fixup log of its own.
+ *
+ * Returns the handle the wallet is to use from here on. The caller keeps
+ * using `src` if this throws: nothing destructive has happened, and the file
+ * still opens as an emulation database.
+ */
+export async function migrateWalletDbToNative(
+ db: Sqlite3Database,
+ src: WalletDbHandle,
+): Promise<NativeMigrationResult> {
+ const ndb = await openNativeSqliteWalletDb(db);
+ const dst = new SqliteWalletDbHandle(ndb);
+ const txc = ndb.txc;
+
+ const previous = await readNativeMigrationInfo(db);
+ if (previous?.status === "complete") {
+ throw Error(
+ "this wallet database has already been migrated to the native schema",
+ );
+ }
+ if (previous?.status === "rolled-back") {
+ throw Error(
+ "this wallet database was rolled back to the IndexedDB schema and is" +
+ " not migrated again automatically",
+ );
+ }
+ if (previous?.status === "running") {
+ // A previous attempt died before the renames. Its partial copy is in the
+ // native tables and nothing references it, so it goes.
+ logger.warn(
+ "discarding the partial copy left by an interrupted migration attempt",
+ );
+ await clearNativeSqliteWalletDb(ndb);
+ }
+
+ const startedAt = nowMicros();
+ await ndb.lock.run(() =>
+ inTransaction(txc, async () => {
+ await (
+ await db.prepare(
+ "INSERT INTO idb_migration (id, status, started_at)" +
+ " VALUES (1, 'running', $started_at)" +
+ " ON CONFLICT (id) DO UPDATE SET status = 'running'," +
+ " started_at = $started_at, finished_at = NULL," +
+ " records_copied = NULL, backup_status = NULL," +
+ " backup_expires_at = NULL",
+ )
+ ).run({ started_at: startedAt });
+ }),
+ );
+
+ logger.info("migrating the wallet database to the native schema");
+ // Verifies its own copy record by record and throws on any difference, so
+ // reaching the next statement means the native tables hold the wallet.
+ const report = await convertWalletDb(src, dst);
+
+ const finishedAt = nowMicros();
+ const backupExpiresAt = finishedAt + retentionMicros();
+
+ // One transaction: the renames and the record of them being done cannot come
+ // apart. A crash between them would leave a file whose emulation tables are
+ // gone and whose bookkeeping still says the emulation is authoritative, and
+ // the retry would then wipe the only remaining copy.
+ await ndb.lock.run(() =>
+ inTransaction(txc, async () => {
+ for (const table of IDB_EMULATION_TABLES) {
+ await (
+ await db.prepare(
+ `ALTER TABLE "${table}" RENAME TO "${backupTableName(table)}"`,
+ )
+ ).run({});
+ }
+ await (
+ await db.prepare(
+ "UPDATE idb_migration SET status = 'complete'," +
+ " finished_at = $finished_at, records_copied = $records_copied," +
+ " backup_status = 'retained'," +
+ " backup_expires_at = $backup_expires_at WHERE id = 1",
+ )
+ ).run({
+ finished_at: finishedAt,
+ records_copied: report.totalRecords,
+ backup_expires_at: backupExpiresAt,
+ });
+ }),
+ );
+
+ logger.info(
+ `migrated ${report.totalRecords} records to the native schema;` +
+ ` the previous database is kept in this file until` +
+ ` ${new Date(backupExpiresAt / 1000).toISOString()}`,
+ );
+
+ // The bookkeeping is reported from what was just written rather than read
+ // back: past the transaction above the emulation's tables are gone, so a
+ // caller that treats a throw as "nothing happened, keep using the old
+ // handle" would be wrong from here on. Nothing after this can throw.
+ return {
+ handle: dst,
+ report,
+ info: {
+ status: "complete",
+ startedAt,
+ finishedAt,
+ recordsCopied: report.totalRecords,
+ backupStatus: "retained",
+ backupExpiresAt,
+ },
+ };
+}
+
+/**
+ * Drop the retained emulation tables once their retention has passed.
+ *
+ * Called when a migrated database is opened, which is the only moment at
+ * which nothing is using it and a schema change is free. Returns whether it
+ * dropped anything.
+ */
+export async function dropExpiredMigrationBackup(
+ db: Sqlite3Database,
+ now: number = nowMicros(),
+): Promise<boolean> {
+ const info = await readNativeMigrationInfo(db);
+ if (info?.status !== "complete" || info.backupStatus !== "retained") {
+ return false;
+ }
+ if (info.backupExpiresAt == null || now < info.backupExpiresAt) {
+ return false;
+ }
+ logger.info("dropping the retained pre-migration database tables");
+ const txc = await SqliteTxControl.create(db);
+ await inTransaction(txc, async () => {
+ for (const table of IDB_EMULATION_TABLES) {
+ await (
+ await db.prepare(`DROP TABLE IF EXISTS "${backupTableName(table)}"`)
+ ).run({});
+ }
+ await (
+ await db.prepare(
+ "UPDATE idb_migration SET backup_status = 'dropped' WHERE id = 1",
+ )
+ ).run({});
+ });
+ return true;
+}
+
+/**
+ * Undo a migration, putting the retained emulation tables back in place.
+ *
+ * The wallet database must not be open: this renames the tables both backends
+ * read. Afterwards the file is an emulation database again and the native
+ * tables are empty, so it opens the way it did before the migration.
+ *
+ * Deliberately not automatic. A wallet that migrated and then misbehaved has
+ * no way to tell whether the migration caused it, and rolling back on its own
+ * would discard whatever the wallet did since -- the emulation tables stopped
+ * being written the moment the migration completed.
+ */
+export async function restoreMigrationBackup(
+ db: Sqlite3Database,
+): Promise<void> {
+ const info = await readNativeMigrationInfo(db);
+ if (info?.status !== "complete") {
+ throw Error("this database was not migrated to the native schema");
+ }
+ if (info.backupStatus !== "retained") {
+ throw Error(
+ `the pre-migration tables are not available (backup is` +
+ ` ${info.backupStatus ?? "absent"})`,
+ );
+ }
+ const ndb = await openNativeSqliteWalletDb(db);
+ // Everything the native tables hold came from the backup or was written
+ // after the migration; either way it is not what the restored database is
+ // supposed to contain.
+ await clearNativeSqliteWalletDb(ndb);
+ await inTransaction(ndb.txc, async () => {
+ for (const table of IDB_EMULATION_TABLES) {
+ await (
+ await db.prepare(
+ `ALTER TABLE "${backupTableName(table)}" RENAME TO "${table}"`,
+ )
+ ).run({});
+ }
+ await (
+ await db.prepare(
+ "UPDATE idb_migration SET backup_status = 'restored'," +
+ " status = 'rolled-back' WHERE id = 1",
+ )
+ ).run({});
+ });
+ logger.info("restored the pre-migration wallet database");
+}
diff --git a/packages/taler-wallet-core/src/db-sqlite-schema.ts b/packages/taler-wallet-core/src/db-sqlite-schema.ts
@@ -88,6 +88,54 @@
export const SQLITE_SCHEMA_VERSION = 5;
/**
+ * Tables of the IndexedDB emulation, children before parents.
+ *
+ * A migrated wallet keeps them in the same file, renamed out of the way, so
+ * the native schema has to know their names: the emulation creates them with
+ * IF NOT EXISTS, and a file it opened after they were renamed would look like
+ * a brand-new, empty wallet rather than like a mistake.
+ *
+ * The order is the one in which they can be dropped with foreign keys
+ * enforced: index_data and unique_index_data reference indexes, indexes
+ * references object_stores, object_stores references databases.
+ */
+export const IDB_EMULATION_TABLES = [
+ "index_data",
+ "unique_index_data",
+ "object_data",
+ "indexes",
+ "object_stores",
+ "databases",
+];
+
+/** Prefix the migration renames the emulation's tables to. */
+export const IDB_BACKUP_PREFIX = "idb_backup_";
+
+/**
+ * Tables that live in the file but are not wallet data.
+ *
+ * Both describe the file rather than the wallet: schema_migrations says which
+ * schema changes ran, idb_migration says where the data came from. Restoring
+ * either from a backup would state something untrue about the file it was
+ * restored into.
+ */
+export const NON_DATA_TABLES = ["schema_migrations", "idb_migration"];
+
+/**
+ * SQL condition selecting the tables that hold wallet data.
+ *
+ * Written once and used by export, import and clear alike: each of them
+ * enumerates tables from sqlite_master, and one of them forgetting the
+ * emulation's retained backup would silently destroy or export it.
+ */
+export const DATA_TABLES_CONDITION = `type = 'table'
+ AND name NOT LIKE 'sqlite_%'
+ AND name NOT LIKE '${IDB_BACKUP_PREFIX}%'
+ AND name NOT IN (${[...NON_DATA_TABLES, ...IDB_EMULATION_TABLES]
+ .map((n) => `'${n}'`)
+ .join(", ")})`;
+
+/**
* A single, ordered schema evolution step.
*
* Replaces both mechanisms the IndexedDB backend needs (versionAdded for
@@ -255,6 +303,38 @@ CREATE TABLE IF NOT EXISTS schema_migrations (
applied_at INTEGER NOT NULL
);
+-- State of the one-way migration from the IndexedDB emulation, which happens
+-- inside this same file.
+--
+-- The table exists in every native database; a row exists only where this
+-- database was produced by migrating an emulation database in place. That
+-- row is what decides which backend a file is opened with: the emulation's
+-- tables are still present under their idb_backup_ names, and the emulation
+-- would recreate the originals empty rather than report that they are gone.
+--
+-- Not wallet data: excluded from export, import and clear alike, because it
+-- describes this file's history and not the wallet's contents.
+CREATE TABLE IF NOT EXISTS idb_migration (
+ -- One row, ever.
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ -- 'running', 'complete' or 'rolled-back'. 'running' means an attempt was
+ -- interrupted before the emulation tables were renamed away, so those
+ -- tables are still the authoritative copy and the native tables are a
+ -- partial write. 'rolled-back' means they were deliberately put back.
+ status TEXT NOT NULL,
+ started_at INTEGER NOT NULL,
+ finished_at INTEGER,
+ -- Records copied, for the log.
+ records_copied INTEGER,
+ -- 'retained', 'dropped' or 'restored': what became of the renamed
+ -- emulation tables.
+ backup_status TEXT,
+ -- After this time the retained backup tables may be dropped. Keeping them
+ -- for a while is what makes a migration that succeeded but produced a
+ -- broken wallet recoverable.
+ backup_expires_at INTEGER
+);
+
-- Config is a key to JSON mapping. The DAL narrows the value type by key,
-- so typed columns would buy nothing.
CREATE TABLE IF NOT EXISTS config (
diff --git a/packages/taler-wallet-core/src/dbtx-handle-impl.ts b/packages/taler-wallet-core/src/dbtx-handle-impl.ts
@@ -75,6 +75,13 @@ export class IdbWalletDbHandle implements WalletDbHandle {
readBackupJson?: (path: string) => Promise<any>;
getDiagnosticStats?: () => unknown;
+ /**
+ * In-place migration to the native schema, set by the host when the
+ * emulation runs over a sqlite database the host can also open natively.
+ * See {@link WalletDbHandle.migrateToNative}.
+ */
+ migrateToNative?: () => Promise<WalletDbHandle>;
+
setNotificationSink(sink: (n: WalletNotification) => void): void {
this.notify = sink;
}
diff --git a/packages/taler-wallet-core/src/dbtx-handle.ts b/packages/taler-wallet-core/src/dbtx-handle.ts
@@ -107,6 +107,20 @@ export interface WalletDbHandle {
readBackupJson?(path: string): Promise<any>;
/**
+ * Migrate this database in place to the native schema and return the handle
+ * to use from here on.
+ *
+ * Present only where the host keeps the wallet in a sqlite file it can open
+ * with either schema, which is every host except a browser extension: there
+ * IndexedDB is the real thing rather than an emulation over sqlite, and
+ * there is nothing to migrate to.
+ *
+ * This handle is unusable afterwards. On failure it is untouched and still
+ * the authoritative database.
+ */
+ migrateToNative?(): Promise<WalletDbHandle>;
+
+ /**
* Backend-specific counters for the testing API.
*
* Deliberately untyped: this is diagnostic output whose shape follows
diff --git a/packages/taler-wallet-core/src/dbtx-sqlite.ts b/packages/taler-wallet-core/src/dbtx-sqlite.ts
@@ -120,6 +120,7 @@ import {
timestampProtocolToDb,
} from "./db-common.js";
import {
+ DATA_TABLES_CONDITION,
SQLITE_BASELINE_SCHEMA,
SchemaMigration,
SQLITE_SCHEMA_VERSION,
@@ -3582,9 +3583,7 @@ export class SqliteWalletTransaction implements WalletDbTransaction {
...(row.timestamp_expired != null
? { timestampExpired: dbTimestamp(row.timestamp_expired) }
: undefined),
- ...(row.taler_uri != null
- ? { talerUri: str(row.taler_uri) }
- : undefined),
+ ...(row.taler_uri != null ? { talerUri: str(row.taler_uri) } : undefined),
};
}
@@ -5132,8 +5131,7 @@ export async function clearNativeSqliteWalletDb(
await ndb.lock.run(async () => {
const rows = await (
await ndb.db.prepare(
- "SELECT name FROM sqlite_master WHERE type = 'table'" +
- " AND name NOT LIKE 'sqlite_%' AND name != 'schema_migrations'",
+ `SELECT name FROM sqlite_master WHERE ${DATA_TABLES_CONDITION}`,
)
).getAll();
await ndb.txc.begin();
@@ -5157,16 +5155,13 @@ export async function clearNativeSqliteWalletDb(
/**
* Names of the tables holding wallet data, in a stable order.
*
- * schema_migrations is excluded: it describes the schema, not the data, and
- * restoring it from a backup could claim migrations were applied that were
- * not.
+ * See DATA_TABLES_CONDITION for what is left out and why.
*/
async function listDataTables(ndb: NativeSqliteWalletDb): Promise<string[]> {
const rows = await (
await ndb.db.prepare(
- "SELECT name FROM sqlite_master WHERE type = 'table'" +
- " AND name NOT LIKE 'sqlite_%' AND name != 'schema_migrations'" +
- " ORDER BY name",
+ `SELECT name FROM sqlite_master WHERE ${DATA_TABLES_CONDITION}` +
+ ` ORDER BY name`,
)
).getAll();
return rows.map((r) => r.name as string);
diff --git a/packages/taler-wallet-core/src/host-impl.node.ts b/packages/taler-wallet-core/src/host-impl.node.ts
@@ -25,6 +25,7 @@
import {
BridgeIDBFactory,
createSqliteBackend,
+ createSqliteBackendOverDb,
shimIndexedDB,
} from "@gnu-taler/idb-bridge";
import { createNodeHelperSqlite3Impl } from "@gnu-taler/idb-bridge/node-helper-sqlite3-impl";
@@ -40,13 +41,19 @@ import {
DefaultNodeWalletArgs,
getSqlite3FilenameFromStoragePath,
} from "./host-common.js";
-import {
- NativeSqliteWalletDb,
- openNativeSqliteWalletDb,
-} from "./dbtx-sqlite.js";
+import { openNativeSqliteWalletDb } from "./dbtx-sqlite.js";
+import { convertWalletDb, DbConversionReport } from "./db-converter.js";
import { Wallet } from "./wallet.js";
import { WalletDbHandle } from "./dbtx-handle.js";
-import { convertWalletDb, DbConversionReport } from "./db-converter.js";
+import {
+ dropExpiredMigrationBackup,
+ inspectWalletDbFile,
+ migrateWalletDbToNative,
+ NativeMigrationInfo,
+ readNativeMigrationInfo,
+ restoreMigrationBackup,
+ WalletDbFileKind,
+} from "./db-native-migration.js";
import * as fs from "node:fs";
import { IdbWalletDbHandle, SqliteWalletDbHandle } from "./dbtx-handle-impl.js";
@@ -61,24 +68,42 @@ async function makeSqliteDb(
args.persistentStoragePath,
);
+ // One connection for both schemas: a migration reads one and writes the
+ // other in this same file, and a second connection to it would bring its
+ // own transaction state and its own locks.
+ const imp = await createNodeHelperSqlite3Impl();
+ const db = await imp.open(dbFilename);
+
// Exactly one backend is opened. Opening both meant every operation that
// works on the whole database had to pick one, and picking wrong produced a
// valid empty database rather than an error.
- if (process.env.TALER_WALLET_NATIVE_DB) {
+ //
+ // Which schema the file's records are already in decides that; the
+ // environment variable only says what a database that does not exist yet
+ // should be created as.
+ const kind = await inspectWalletDbFile(db);
+ if (
+ kind === "native" ||
+ (kind === "empty" && process.env.TALER_WALLET_NATIVE_DB)
+ ) {
logger.info(`using NATIVE sqlite3 wallet DB at ${dbFilename}`);
logger.warn("the native sqlite3 wallet DB backend is experimental");
- const nativeImp = await createNodeHelperSqlite3Impl();
- const ndb = await openNativeSqliteWalletDb(
- await nativeImp.open(dbFilename),
- );
+ // Opening is the moment nothing holds the database, which is what
+ // dropping tables needs.
+ await dropExpiredMigrationBackup(db);
+ const ndb = await openNativeSqliteWalletDb(db);
return new SqliteWalletDbHandle(ndb);
}
+ if (process.env.TALER_WALLET_NATIVE_DB) {
+ logger.warn(
+ `${dbFilename} holds an IndexedDB-emulation wallet database; opening` +
+ ` it natively would show an empty wallet. Set` +
+ ` TALER_WALLET_MIGRATE_NATIVE_DB=1 to convert it instead.`,
+ );
+ }
logger.info(`using database ${dbFilename}`);
- const imp = await createNodeHelperSqlite3Impl();
- const myBackend = await createSqliteBackend(imp, {
- filename: dbFilename,
- });
+ const myBackend = await createSqliteBackendOverDb(imp, db);
myBackend.enableTracing = tracing;
if (process.env.TALER_WALLET_STATS) {
myBackend.trackStats = true;
@@ -93,10 +118,58 @@ async function makeSqliteDb(
return { path };
};
handle.getDiagnosticStats = () => myBackend.accessStats;
+ handle.migrateToNative = async () =>
+ (await migrateWalletDbToNative(db, handle)).handle;
return handle;
}
/**
+ * What a wallet database file on disk holds and where it came from.
+ *
+ * Opens the file directly rather than through a wallet: the questions it
+ * answers -- which schema is authoritative, whether a rollback is still
+ * possible -- are about a database nobody should be running.
+ */
+export async function inspectWalletDbPath(dbPath: string): Promise<{
+ kind: WalletDbFileKind;
+ migration: NativeMigrationInfo | undefined;
+}> {
+ if (!fs.existsSync(dbPath)) {
+ throw Error(`wallet database ${dbPath} does not exist`);
+ }
+ const imp = await createNodeHelperSqlite3Impl();
+ const db = await imp.open(dbPath);
+ try {
+ return {
+ kind: await inspectWalletDbFile(db),
+ migration: await readNativeMigrationInfo(db),
+ };
+ } finally {
+ await db.close();
+ }
+}
+
+/**
+ * Undo an in-place migration, from the copy it retained.
+ *
+ * Everything the wallet did since the migration is in the native tables and
+ * stays there; what comes back is the database as it was the moment before
+ * the migration ran. The wallet must not be running.
+ */
+export async function rollbackWalletDbMigration(dbPath: string): Promise<void> {
+ if (!fs.existsSync(dbPath)) {
+ throw Error(`wallet database ${dbPath} does not exist`);
+ }
+ const imp = await createNodeHelperSqlite3Impl();
+ const db = await imp.open(dbPath);
+ try {
+ await restoreMigrationBackup(db);
+ } finally {
+ await db.close();
+ }
+}
+
+/**
* Get a wallet instance with default settings for node.
*
* Extended version that allows getting DB stats.
diff --git a/packages/taler-wallet-core/src/host-impl.qtart.ts b/packages/taler-wallet-core/src/host-impl.qtart.ts
@@ -29,7 +29,7 @@ import type {
} from "@gnu-taler/idb-bridge";
import {
BridgeIDBFactory,
- createSqliteBackend,
+ createSqliteBackendOverDb,
shimIndexedDB,
} from "@gnu-taler/idb-bridge";
import {
@@ -46,9 +46,15 @@ import {
getSqlite3FilenameFromStoragePath,
} from "./host-common.js";
import { exportDb } from "./db-indexeddb.js";
+import {
+ dropExpiredMigrationBackup,
+ inspectWalletDbFile,
+ migrateWalletDbToNative,
+} from "./db-native-migration.js";
+import { openNativeSqliteWalletDb } from "./dbtx-sqlite.js";
import { Wallet } from "./wallet.js";
import { WalletDbHandle } from "./dbtx-handle.js";
-import { IdbWalletDbHandle } from "./dbtx-handle-impl.js";
+import { IdbWalletDbHandle, SqliteWalletDbHandle } from "./dbtx-handle-impl.js";
const logger = new Logger("host-impl.qtart.ts");
@@ -103,9 +109,20 @@ async function makeSqliteDb(
);
logger.info(`opening sqlite3 database ${j2s(filename)}`);
const imp = await createQtartSqlite3Impl();
- const myBackend = await createSqliteBackend(imp, {
- filename,
- });
+ // One connection for both schemas: a migration reads one and writes the
+ // other in this same file. This is the platform in-place migration exists
+ // for -- the mobile wallets hand wallet-core a database and no directory to
+ // put a second one in.
+ const db = await imp.open(filename);
+
+ const kind = await inspectWalletDbFile(db);
+ if (kind === "native") {
+ logger.info("opening the wallet database with the native schema");
+ await dropExpiredMigrationBackup(db);
+ return new SqliteWalletDbHandle(await openNativeSqliteWalletDb(db));
+ }
+
+ const myBackend = await createSqliteBackendOverDb(imp, db);
myBackend.trackStats = true;
myBackend.enableTracing = false;
const handle = new IdbWalletDbHandle(new BridgeIDBFactory(myBackend), () => ({
@@ -136,6 +153,8 @@ async function makeSqliteDb(
throw Error(`forcing format ${forceFormat} not supported`);
}
};
+ handle.migrateToNative = async () =>
+ (await migrateWalletDbToNative(db, handle)).handle;
handle.readBackupJson = async (path: string): Promise<any> => {
const errObj = { errno: undefined };
const file = qjsStd.open(path, "r", errObj);
diff --git a/packages/taler-wallet-core/src/index.node.ts b/packages/taler-wallet-core/src/index.node.ts
@@ -32,4 +32,16 @@ export { makeIdbRunner, makeSqliteRunner } from "./dbtx-runners.js";
export { convertWalletDbFile } from "./host-impl.node.js";
export { convertWalletDb } from "./db-converter.js";
export type { DbConversionReport } from "./db-converter.js";
+
+// In-place migration to the native schema. Inspecting and rolling one back
+// works on a file, so these are node-only too; performing the migration is
+// not here at all, because it happens when the wallet is initialized.
+export {
+ inspectWalletDbPath,
+ rollbackWalletDbMigration,
+} from "./host-impl.node.js";
+export type {
+ NativeMigrationInfo,
+ WalletDbFileKind,
+} from "./db-native-migration.js";
export type { DbTxRunner } from "./dbtx-conformance.js";
diff --git a/packages/taler-wallet-core/src/requests.ts b/packages/taler-wallet-core/src/requests.ts
@@ -1075,6 +1075,14 @@ async function handleSetWalletRunConfig(
}
wex.ws.initWithConfig(applyRunConfigDefaults(req.config));
+ // Before anything else touches the database: everything below this line
+ // runs against whichever backend the wallet ends up on, and the migration
+ // needs the fixup log of the old one to have been replayed, which opening
+ // it for the write above did.
+ if (wex.ws.config.features.migrateNativeDb) {
+ await wex.ws.migrateDbToNativeSchema();
+ }
+
if (wex.ws.config.testing.skipDefaults) {
logger.trace("skipping defaults");
} else {
diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts
@@ -590,6 +590,29 @@ function coinSelectionAlgorithmFromEnv(): CoinSelectionAlgorithm | undefined {
}
}
+/**
+ * Environment variable that turns on the migration to the native database.
+ *
+ * Configures wallet-core rather than a particular frontend, so it is named
+ * for wallet-core. A client with a settings UI passes the flag in the init
+ * request instead.
+ */
+const migrateNativeDbEnvVar = "TALER_WALLET_MIGRATE_NATIVE_DB";
+
+/**
+ * Read the native-database migration flag from the environment.
+ *
+ * Returns undefined where there is no environment or the variable is unset,
+ * so the caller falls back to its own default.
+ */
+function migrateNativeDbFromEnv(): boolean | undefined {
+ const val = getenv(migrateNativeDbEnvVar);
+ if (val == null || val === "") {
+ return undefined;
+ }
+ return val !== "0";
+}
+
export function applyRunConfigDefaults(
wcp?: PartialWalletRunConfig,
): WalletRunConfig {
@@ -606,6 +629,8 @@ export function applyRunConfigDefaults(
return {
features: {
allowHttp: true,
+ migrateNativeDb:
+ wcp?.features?.migrateNativeDb ?? migrateNativeDbFromEnv() ?? false,
},
testing: {
devModeActive: wcp?.testing?.devModeActive ?? false,
@@ -944,6 +969,58 @@ export class InternalWalletState {
}
/**
+ * Migrate the database to the native schema, if it is not already there and
+ * the host can do it.
+ *
+ * Returns whether the wallet is on the native schema afterwards. A failed
+ * migration is logged and reported, not thrown: the old database is
+ * untouched and the wallet keeps running on it, which is a far better
+ * outcome than refusing to start.
+ *
+ * Runs under the same guard as opening and importing, so no transaction can
+ * be in flight while the handle underneath the wallet is replaced. Every
+ * transaction resolves the handle through this class when it starts, so
+ * work that was queued before the swap runs against the new database.
+ */
+ async migrateDbToNativeSchema(): Promise<boolean> {
+ const oldHandle = this.dbHandle;
+ if (!oldHandle.migrateToNative) {
+ // Either the database is already native or the host cannot migrate it;
+ // both are "nothing to do", and neither is worth an alarming message on
+ // every initialization.
+ logger.trace("this wallet database offers no in-place migration");
+ return false;
+ }
+ if (this.loadingDb) {
+ while (this.loadingDb) {
+ await this.loadingDbCond.wait();
+ }
+ }
+ this.loadingDb = true;
+ try {
+ const newHandle = await oldHandle.migrateToNative();
+ this.dbHandle = newHandle;
+ newHandle.setNotificationSink((n) => this.notify(n));
+ await oldHandle.close();
+ // Records from the old database are in the caches under the same
+ // identities, but nothing guarantees the two backends materialise them
+ // identically, and a stale cached record would outlive the database it
+ // came from.
+ this.clearAllCaches();
+ return true;
+ } catch (e) {
+ logger.error(
+ `migration to the native database failed, continuing with the` +
+ ` existing one: ${safeStringifyException(e)}`,
+ );
+ return false;
+ } finally {
+ this.loadingDb = false;
+ this.loadingDbCond.trigger();
+ }
+ }
+
+ /**
* Prepare database for import by closing it.
*/
async suspendDatabase(): Promise<void> {
