commit 9be8e931ecba22572875a060f4d87db2527993fe parent 5bf17de54e127d90a658e024634bf74dae0957a0 Author: Florian Dold <dold@taler.net> Date: Tue, 1 Sep 2026 21:39:06 +0200 wallet-core: import SQLite database files Diffstat:
19 files changed, 712 insertions(+), 31 deletions(-)
diff --git a/packages/idb-bridge/src/node-helper-sqlite3-impl.ts b/packages/idb-bridge/src/node-helper-sqlite3-impl.ts @@ -452,7 +452,10 @@ export async function createNodeHelperSqlite3Impl( async shutdown(): Promise<void> { await helper.shutdown(); }, - async open(filename: string): Promise<Sqlite3Database> { + async open( + filename: string, + options: { readonly?: boolean; immutable?: boolean } = {}, + ): Promise<Sqlite3Database> { if (enableTracing) { console.error(`opening database ${filename}`); } @@ -461,6 +464,7 @@ export async function createNodeHelperSqlite3Impl( const wr = new Writer(); wr.writeUint16(myDbId); wr.writeString(filename); + wr.writeUint8((options.readonly ? 1 : 0) | (options.immutable ? 2 : 0)); const payload = wr.reap(); const commRes = await helper.communicate(HelperCmd.OPEN, payload); expectCommunicateSuccess(commRes); diff --git a/packages/idb-bridge/src/sqlite3-interface.ts b/packages/idb-bridge/src/sqlite3-interface.ts @@ -64,5 +64,8 @@ export function isSqlite3Error(error: unknown): error is Sqlite3Error { * to be used by our IndexedDB sqlite3 backend. */ export interface Sqlite3Interface { - open(filename: string): Promise<Sqlite3Database>; + open( + filename: string, + options?: { readonly?: boolean; immutable?: boolean }, + ): Promise<Sqlite3Database>; } diff --git a/packages/idb-bridge/taler-helper-sqlite3 b/packages/idb-bridge/taler-helper-sqlite3 @@ -18,6 +18,7 @@ import sqlite3 import sys import os import sys +import urllib.parse if sys.version_info < (3, 10): raise SystemExit( @@ -244,9 +245,20 @@ while True: raise Exception("DB already connected") db_handle = pr.read_uint16() filename = pr.read_string() + open_flags = pr.read_uint8() if pr.pos < len(pr.data) else 0 + readonly = open_flags & 1 != 0 + immutable = open_flags & 2 != 0 # This only works in python>=3.12 # dbconn = sqlite3.connect(filename, autocommit=True, isolation_level=None) - dbconn = sqlite3.connect(filename, isolation_level=None) + if readonly: + uri_filename = urllib.parse.quote(filename, safe="/") + immutable_param = "&immutable=1" if immutable else "" + dbconn = sqlite3.connect( + "file:%s?mode=ro%s" % (uri_filename, immutable_param), + uri=True, + isolation_level=None) + else: + dbconn = sqlite3.connect(filename, isolation_level=None) # Make sure we are not in a transaction dbconn.commit() db_handles[db_handle] = dbconn diff --git a/packages/taler-util/src/qtart.ts b/packages/taler-util/src/qtart.ts @@ -55,6 +55,8 @@ export interface QjsOsLib { setMessageFromHostHandler(h: (s: string) => void): void; rename(oldPath: string, newPath: string): number; remove(path: string): number; + mkdir(path: string, mode?: number): number; + stat(path: string): [unknown, number]; readonly O_RDONLY: number; readonly O_WRONLY: number; diff --git a/packages/taler-util/src/types-taler-wallet.ts b/packages/taler-util/src/types-taler-wallet.ts @@ -4674,7 +4674,7 @@ export interface ExportDbToFileResponse { export interface ImportDbFromFileRequest { /** - * Full path to the backup. + * Full path to a .json or .sqlite3 backup. */ path: string; diff --git a/packages/taler-wallet-cli/src/index.ts b/packages/taler-wallet-cli/src/index.ts @@ -3042,7 +3042,7 @@ backupCli help: "Import a wallet database from a file, replacing existing data.", }) .requiredArgument("path", clk.STRING, { - help: "Path to the backup file to import.", + help: "Path to the .json or .sqlite3 backup file to import.", }) .action(async (args) => { await withWallet(args, { lazyTaskLoop: true }, async (wallet) => { diff --git a/packages/taler-wallet-core/src/db/handle.ts b/packages/taler-wallet-core/src/db/handle.ts @@ -138,9 +138,15 @@ export interface WalletDbHandle { forceFormat?: string, ): Promise<{ path: string }>; - /** Read a dump previously written by exportToFile. Absent if unsupported. */ + /** @deprecated Implement readBackupFile instead. */ readBackupJson?(path: string): Promise<any>; + /** Read a dump previously written by exportToFile. Absent if unsupported. */ + readBackupFile?( + path: string, + options?: WalletDbMigrationOptions, + ): Promise<any>; + /** * Migrate this database in place to the native schema and return the handle * to use from here on. diff --git a/packages/taler-wallet-core/src/db/indexeddb/handle.ts b/packages/taler-wallet-core/src/db/indexeddb/handle.ts @@ -105,7 +105,12 @@ export class IdbWalletDbHandle implements WalletDbHandle { stem: string, forceFormat?: string, ) => Promise<{ path: string }>; + /** @deprecated Implement readBackupFile instead. */ readBackupJson?: (path: string) => Promise<any>; + readBackupFile?: ( + path: string, + options?: WalletDbMigrationOptions, + ) => Promise<any>; getDiagnosticStats?: () => unknown; /** diff --git a/packages/taler-wallet-core/src/db/migration/import.ts b/packages/taler-wallet-core/src/db/migration/import.ts @@ -47,6 +47,7 @@ import { IdbWalletDbHandle } from "../indexeddb/handle.js"; import { openNativeSqliteWalletDb } from "../sqlite/database.js"; import { SqliteWalletDbHandle } from "../sqlite/handle.js"; import { convertWalletDb, DB_CONVERSION_STEP_COUNT } from "./converter.js"; +import { inspectWalletDbFile } from "./native.js"; export function getWalletDbDumpBackend( dump: unknown, @@ -64,6 +65,97 @@ export function getWalletDbDumpBackend( return isIndexedDb ? "indexeddb" : "sqlite"; } +/** + * Read a physical wallet database through a disposable SQLite snapshot. + * + * Opening either wallet backend can upgrade its schema or apply fixups. The + * caller supplies a fresh snapshot path so that this work never changes the + * backup being imported. VACUUM INTO also folds a source WAL into one + * self-contained, transactionally consistent database file. + * + * sourceIsSelfContained is only valid after the host has established that no + * rollback-journal or WAL sidecar exists. It lets SQLite read a clean + * WAL-mode database from a directory where it cannot create a shared-memory + * file, without hiding pending transactions in an existing sidecar. + */ +export async function readWalletDbSqliteFile( + sqlite3Impl: Sqlite3Interface, + sourcePath: string, + snapshotPath: string, + options: WalletDbMigrationOptions = {}, + sourceIsSelfContained = false, +): Promise<unknown> { + const cancellationToken = + options.cancellationToken ?? CancellationToken.CONTINUE; + let sourceDb: Sqlite3Database | undefined; + let snapshotDb: Sqlite3Database | undefined; + let idbHandle: IdbWalletDbHandle | undefined; + let sqliteHandle: SqliteWalletDbHandle | undefined; + try { + cancellationToken.throwIfCancelled(); + sourceDb = await sqlite3Impl.open(sourcePath, { + readonly: true, + immutable: sourceIsSelfContained, + }); + cancellationToken.throwIfCancelled(); + await ( + await sourceDb.prepare("VACUUM INTO $filename") + ).run({ filename: snapshotPath }); + cancellationToken.throwIfCancelled(); + await sourceDb.close(); + sourceDb = undefined; + + snapshotDb = await sqlite3Impl.open(snapshotPath); + const kind = await inspectWalletDbFile(snapshotDb); + cancellationToken.throwIfCancelled(); + if (kind === "ambiguous") { + throw Error( + "wallet database backup contains records in both schemas and has no trustworthy authority marker", + ); + } + if (kind === "empty") { + throw Error("wallet database backup does not contain a wallet schema"); + } + + if (kind === "native") { + sqliteHandle = new SqliteWalletDbHandle( + await openNativeSqliteWalletDb(snapshotDb), + ); + snapshotDb = undefined; + const dump = await sqliteHandle.exportDatabase(); + cancellationToken.throwIfCancelled(); + return dump; + } + + const dbForBackend = snapshotDb; + const idbBackend = await createSqliteBackendOverDb( + sqlite3Impl, + dbForBackend, + ); + idbHandle = new IdbWalletDbHandle( + new BridgeIDBFactory(idbBackend), + undefined, + undefined, + async () => { + try { + await idbBackend.dispose(); + } finally { + await dbForBackend.close(); + } + }, + ); + snapshotDb = undefined; + const dump = await idbHandle.exportDatabase(); + cancellationToken.throwIfCancelled(); + return dump; + } finally { + await idbHandle?.close().catch(() => {}); + await sqliteHandle?.close().catch(() => {}); + await snapshotDb?.close().catch(() => {}); + await sourceDb?.close().catch(() => {}); + } +} + export async function importWalletDbDump( sqlite3Impl: Sqlite3Interface, target: WalletDbHandle, diff --git a/packages/taler-wallet-core/src/db/migration/native.test.ts b/packages/taler-wallet-core/src/db/migration/native.test.ts @@ -66,6 +66,7 @@ import { initSqliteWalletDb, openNativeSqliteWalletDb, } from "../sqlite/database.js"; +import { SqliteWalletDbHandle } from "../sqlite/handle.js"; import { createNativeWalletHost2, inspectWalletDbPath, @@ -150,6 +151,21 @@ async function makeMinimalIdbDb(filename = ":memory:"): Promise<{ return { db, handle }; } +async function makeMinimalNativeDb(filename: string): Promise<void> { + const imp = await createNodeHelperSqlite3Impl({ enableTracing: false }); + const handle = new SqliteWalletDbHandle( + await openNativeSqliteWalletDb(await imp.open(filename)), + ); + try { + await handle.runReadWriteTx((tx) => + tx.upsertConfig({ key: "fault-test" as any, value: 1 }), + ); + } finally { + await handle.close(); + await imp.shutdown(); + } +} + test("wallet database ownership excludes another SQLite connection", async () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "wallet-db-owner-")); const filename = path.join(directory, "wallet.sqlite3"); @@ -809,6 +825,224 @@ test("useNativeDb initializes empty storage directly as native", async () => { } }); +for (const sourceBackend of ["indexeddb", "sqlite"] as const) { + for (const targetBackend of ["indexeddb", "sqlite"] as const) { + const sourceLabel = + sourceBackend === "sqlite" ? "native" : "IndexedDB-emulation"; + test(`file import restores ${sourceLabel} sqlite into ${targetBackend}`, async () => { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), "wallet-sqlite-import-"), + ); + const sourcePath = path.join(dir, "source.sqlite3"); + const targetPath = path.join(dir, "target.sqlite3"); + let wallet: + | Awaited<ReturnType<typeof createNativeWalletHost2>>["wallet"] + | undefined; + try { + if (sourceBackend === "indexeddb") { + const source = await makeMinimalIdbDb(sourcePath); + await source.handle.close(); + await source.db.close(); + } else { + await makeMinimalNativeDb(sourcePath); + } + const sourceBefore = fs.readFileSync(sourcePath); + + ({ wallet } = await createNativeWalletHost2({ + persistentStoragePath: targetPath, + })); + const init = await wallet.client.call(WalletApiOperation.InitWallet, { + config: { + lazyTaskLoop: true, + testing: { skipDefaults: true }, + features: { useNativeDb: targetBackend === "sqlite" }, + }, + }); + assert.strictEqual(init.databaseBackend, targetBackend); + await wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: sourcePath, + progressToken: `${sourceBackend}-file-to-${targetBackend}`, + }); + const imported = await wallet.client.call( + WalletApiOperation.ExportDb, + {}, + ); + assert.match(JSON.stringify(imported), /fault-test/); + assert.deepStrictEqual(fs.readFileSync(sourcePath), sourceBefore); + } finally { + if (wallet) { + await wallet.client + .call(WalletApiOperation.Shutdown, {}) + .catch(() => {}); + } + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + } +} + +test("file import uses temporary storage outside a read-only source directory", async () => { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), "wallet-sqlite-import-readonly-"), + ); + const sourceDirectory = path.join(dir, "source"); + const targetDirectory = path.join(dir, "target"); + const temporaryStoragePath = path.join(dir, "temporary"); + fs.mkdirSync(sourceDirectory); + fs.mkdirSync(targetDirectory); + fs.mkdirSync(temporaryStoragePath); + const sourcePath = path.join(sourceDirectory, "source.sqlite3"); + const targetPath = path.join(targetDirectory, "target.sqlite3"); + let wallet: + | Awaited<ReturnType<typeof createNativeWalletHost2>>["wallet"] + | undefined; + try { + await makeMinimalNativeDb(sourcePath); + const sourceBefore = fs.readFileSync(sourcePath); + fs.chmodSync(sourceDirectory, 0o555); + + ({ wallet } = await createNativeWalletHost2({ + persistentStoragePath: targetPath, + temporaryStoragePath, + })); + await wallet.client.call(WalletApiOperation.InitWallet, { + config: { lazyTaskLoop: true, testing: { skipDefaults: true } }, + }); + await wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: sourcePath, + }); + + const imported = await wallet.client.call(WalletApiOperation.ExportDb, {}); + assert.match(JSON.stringify(imported), /fault-test/); + assert.deepStrictEqual(fs.readFileSync(sourcePath), sourceBefore); + assert.deepStrictEqual(fs.readdirSync(temporaryStoragePath), []); + } finally { + if (wallet) { + await wallet.client.call(WalletApiOperation.Shutdown, {}).catch(() => {}); + } + fs.chmodSync(sourceDirectory, 0o755); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("file import falls back to persistent storage when temporary storage is unavailable", async () => { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), "wallet-sqlite-import-fallback-"), + ); + const sourceDirectory = path.join(dir, "source"); + fs.mkdirSync(sourceDirectory); + const sourcePath = path.join(sourceDirectory, "source.sqlite3"); + const targetPath = path.join(dir, "target.sqlite3"); + let wallet: + | Awaited<ReturnType<typeof createNativeWalletHost2>>["wallet"] + | undefined; + try { + await makeMinimalNativeDb(sourcePath); + fs.chmodSync(sourceDirectory, 0o555); + ({ wallet } = await createNativeWalletHost2({ + persistentStoragePath: targetPath, + temporaryStoragePath: path.join(dir, "not-present"), + })); + await wallet.client.call(WalletApiOperation.InitWallet, { + config: { lazyTaskLoop: true, testing: { skipDefaults: true } }, + }); + await wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: sourcePath, + }); + + const imported = await wallet.client.call(WalletApiOperation.ExportDb, {}); + assert.match(JSON.stringify(imported), /fault-test/); + assert.deepStrictEqual(fs.readdirSync(sourceDirectory), ["source.sqlite3"]); + } finally { + if (wallet) { + await wallet.client.call(WalletApiOperation.Shutdown, {}).catch(() => {}); + } + fs.chmodSync(sourceDirectory, 0o755); + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test("file import rejects invalid sqlite backups without replacing the wallet", async () => { + const dir = fs.mkdtempSync( + path.join(os.tmpdir(), "wallet-invalid-sqlite-import-"), + ); + const targetPath = path.join(dir, "target.sqlite3"); + let wallet: + | Awaited<ReturnType<typeof createNativeWalletHost2>>["wallet"] + | undefined; + try { + ({ wallet } = await createNativeWalletHost2({ + persistentStoragePath: targetPath, + })); + await wallet.client.call(WalletApiOperation.InitWallet, { + config: { lazyTaskLoop: true, testing: { skipDefaults: true } }, + }); + const before = await wallet.client.call(WalletApiOperation.ExportDb, {}); + + const wrongExtension = path.join(dir, "backup.db"); + fs.writeFileSync(wrongExtension, "not a backup"); + await assert.rejects( + wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: wrongExtension, + }), + (error: unknown) => + error instanceof TalerError && + error.errorDetail.code === TalerErrorCode.WALLET_CORE_API_BAD_REQUEST, + ); + + const corrupt = path.join(dir, "corrupt.sqlite3"); + fs.writeFileSync(corrupt, "not a sqlite database"); + await assert.rejects( + wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: corrupt, + }), + /database|sqlite/i, + ); + + const empty = path.join(dir, "empty.sqlite3"); + const emptyImpl = await createNodeHelperSqlite3Impl({ + enableTracing: false, + }); + await (await emptyImpl.open(empty)).close(); + await emptyImpl.shutdown(); + await assert.rejects( + wallet.client.call(WalletApiOperation.ImportDbFromFile, { path: empty }), + /does not contain a wallet schema/, + ); + + const ambiguous = path.join(dir, "ambiguous.sqlite3"); + const mixed = await makeMinimalIdbDb(ambiguous); + await openNativeSqliteWalletDb(mixed.db); + await ( + await mixed.db.prepare( + "INSERT INTO config (key, value) VALUES ('native-only', '\"mixed\"')", + ) + ).run({}); + await mixed.handle.close(); + await mixed.db.close(); + assert.strictEqual( + (await inspectWalletDbPath(ambiguous)).kind, + "ambiguous", + ); + await assert.rejects( + wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: ambiguous, + }), + /records in both schemas/, + ); + + assert.deepStrictEqual( + await wallet.client.call(WalletApiOperation.ExportDb, {}), + before, + ); + } finally { + if (wallet) { + await wallet.client.call(WalletApiOperation.Shutdown, {}).catch(() => {}); + } + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + test("useNativeDb leaves existing IndexedDB for explicit migration", async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "wallet-native-explicit-")); const dbPath = path.join(dir, "wallet.sqlite3"); diff --git a/packages/taler-wallet-core/src/host-common.ts b/packages/taler-wallet-core/src/host-common.ts @@ -34,6 +34,16 @@ export interface DefaultNodeWalletArgs { persistentStoragePath?: string; /** + * Directory for disposable files created by filesystem-backed wallet + * operations. + * + * Native hosts should pass their platform-provided temporary directory. + * Filesystem operations fall back to the persistent database directory, and + * finally beside their input if neither directory can be used. + */ + temporaryStoragePath?: string; + + /** * Handler for asynchronous notifications from the wallet. */ notifyHandler?: (n: WalletNotification) => void; diff --git a/packages/taler-wallet-core/src/host-impl.node.ts b/packages/taler-wallet-core/src/host-impl.node.ts @@ -58,16 +58,22 @@ import { WalletDbFileKind, } from "./db/migration/native.js"; import * as fs from "node:fs"; +import * as path from "node:path"; import { IdbWalletDbHandle } from "./db/indexeddb/handle.js"; import { SqliteWalletDbHandle } from "./db/sqlite/handle.js"; import { getWalletDbDumpBackend, importWalletDbDump, + readWalletDbSqliteFile, } from "./db/migration/import.js"; const logger = new Logger("host-impl.node.ts"); -function addNodeDatabaseCapabilities(handle: WalletDbHandle): WalletDbHandle { +function addNodeDatabaseCapabilities( + handle: WalletDbHandle, + temporaryStoragePath: string | undefined, + persistentStoragePath: string | undefined, +): WalletDbHandle { const exportPhysical = handle.exportToFile?.bind(handle); handle.exportToFile = async (directory, stem, forceFormat) => { if (forceFormat === "json") { @@ -87,8 +93,77 @@ function addNodeDatabaseCapabilities(handle: WalletDbHandle): WalletDbHandle { } return await exportPhysical(directory, stem, "sqlite3"); }; - handle.readBackupJson = async (path) => - JSON.parse(await fs.promises.readFile(path, "utf-8")); + handle.readBackupJson = async (backupPath) => + JSON.parse(await fs.promises.readFile(backupPath, "utf-8")); + handle.readBackupFile = async (backupPath, options) => { + if (backupPath.endsWith(".json")) { + options?.cancellationToken?.throwIfCancelled(); + const dump = await handle.readBackupJson!(backupPath); + options?.cancellationToken?.throwIfCancelled(); + return dump; + } + if (!backupPath.endsWith(".sqlite3")) { + throw Error("DB file import only supports .json and .sqlite3 files"); + } + await fs.promises.access(backupPath, fs.constants.R_OK); + let temporaryDirectory: string | undefined; + const temporaryDirectoryRoots = [ + temporaryStoragePath, + persistentStoragePath ? path.dirname(persistentStoragePath) : undefined, + path.dirname(backupPath), + ].filter( + (candidate, index, candidates): candidate is string => + candidate !== undefined && candidates.indexOf(candidate) === index, + ); + for (const [index, root] of temporaryDirectoryRoots.entries()) { + try { + temporaryDirectory = await fs.promises.mkdtemp( + path.join(root, "taler-wallet-import-"), + ); + break; + } catch { + if (index === temporaryDirectoryRoots.length - 1) { + throw Error("could not create a temporary database directory"); + } + } + } + if (!temporaryDirectory) { + throw Error("could not create a temporary database directory"); + } + const snapshotPath = path.join(temporaryDirectory, "snapshot.sqlite3"); + const sourceHasSidecar = await Promise.all( + ["-journal", "-wal"].map((suffix) => + fs.promises + .access(`${backupPath}${suffix}`) + .then(() => true) + .catch(() => false), + ), + ).then((results) => results.some(Boolean)); + let temporarySqlite: + | Awaited<ReturnType<typeof createNodeHelperSqlite3Impl>> + | undefined; + try { + temporarySqlite = await createNodeHelperSqlite3Impl({ + enableTracing: false, + }); + return await readWalletDbSqliteFile( + temporarySqlite, + backupPath, + snapshotPath, + options, + !sourceHasSidecar, + ); + } finally { + try { + await temporarySqlite?.shutdown(); + } finally { + await fs.promises.rm(temporaryDirectory, { + recursive: true, + force: true, + }); + } + } + }; handle.importAnyDatabase = async (dump, finalize, options) => { if (getWalletDbDumpBackend(dump) === handle.name) { await handle.importDatabase(dump, finalize, options); @@ -157,7 +232,11 @@ async function makeSqliteDb( // dropping tables needs. await dropExpiredMigrationBackup(db); const ndb = await openNativeSqliteWalletDb(db); - return addNodeDatabaseCapabilities(new SqliteWalletDbHandle(ndb)); + return addNodeDatabaseCapabilities( + new SqliteWalletDbHandle(ndb), + args.temporaryStoragePath, + dbFilename, + ); } if (process.env.TALER_WALLET_NATIVE_DB) { logger.warn( @@ -195,16 +274,28 @@ async function makeSqliteDb( handle.migrateToNative = async (options) => { const result = await migrateWalletDbToNative(db, handle, options); connectionTransferred = true; - return addNodeDatabaseCapabilities(result.handle); + return addNodeDatabaseCapabilities( + result.handle, + args.temporaryStoragePath, + dbFilename, + ); }; if (kind === "empty") { handle.openNativeIfEmpty = async () => { const result = await openNativeWalletDbForEmptyStorage(db); connectionTransferred = true; - return addNodeDatabaseCapabilities(result); + return addNodeDatabaseCapabilities( + result, + args.temporaryStoragePath, + dbFilename, + ); }; } - return addNodeDatabaseCapabilities(handle); + return addNodeDatabaseCapabilities( + handle, + args.temporaryStoragePath, + dbFilename, + ); } /** diff --git a/packages/taler-wallet-core/src/host-impl.qtart.test.ts b/packages/taler-wallet-core/src/host-impl.qtart.test.ts @@ -0,0 +1,49 @@ +/* + This file is part of GNU Taler + (C) 2026 Taler Systems S.A. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> + */ + +import assert from "node:assert"; +import { test } from "node:test"; +import { + createQtartSqlite3Impl, + getQtartApiVersion, + REQUIRED_QTART_API_VERSION, +} from "./host-impl.qtart.js"; + +test("Qtart API version rejects runtimes without SQLite open options", async () => { + const globalWithTart = globalThis as typeof globalThis & { _tart?: unknown }; + const previousTart = globalWithTart._tart; + try { + delete globalWithTart._tart; + assert.strictEqual(getQtartApiVersion(), 0); + + globalWithTart._tart = {}; + assert.strictEqual(getQtartApiVersion(), 0); + await assert.rejects( + createQtartSqlite3Impl(), + new RegExp(`Qtart API version ${REQUIRED_QTART_API_VERSION}`), + ); + + globalWithTart._tart = { apiVersion: REQUIRED_QTART_API_VERSION }; + assert.strictEqual(getQtartApiVersion(), REQUIRED_QTART_API_VERSION); + await createQtartSqlite3Impl(); + } finally { + if (previousTart === undefined) { + delete globalWithTart._tart; + } else { + globalWithTart._tart = previousTart; + } + } +}); diff --git a/packages/taler-wallet-core/src/host-impl.qtart.ts b/packages/taler-wallet-core/src/host-impl.qtart.ts @@ -33,13 +33,15 @@ import { shimIndexedDB, } from "@gnu-taler/idb-bridge"; import { + encodeCrock, + getRandomBytes, j2s, Logger, SetTimeoutTimerAPI, WalletRunConfig, } from "@gnu-taler/taler-util"; import { createPlatformHttpLib } from "@gnu-taler/taler-util/http"; -import { qjsStd } from "@gnu-taler/taler-util/qtart"; +import { qjsOs, qjsStd } from "@gnu-taler/taler-util/qtart"; import { SynchronousCryptoWorkerFactoryPlain } from "./crypto/workers/synchronousWorkerFactoryPlain.js"; import { acquireSqliteWalletDbOwnership, @@ -57,13 +59,29 @@ import { Wallet } from "./wallet.js"; import { WalletDbHandle } from "./db/handle.js"; import { IdbWalletDbHandle } from "./db/indexeddb/handle.js"; import { SqliteWalletDbHandle } from "./db/sqlite/handle.js"; -import { importWalletDbDump } from "./db/migration/import.js"; +import { + importWalletDbDump, + readWalletDbSqliteFile, +} from "./db/migration/import.js"; const logger = new Logger("host-impl.qtart.ts"); let numStmt = 0; -function addQtartDatabaseCapabilities(handle: WalletDbHandle): WalletDbHandle { +/** Qtart API version that first implements SQLite open options. */ +export const REQUIRED_QTART_API_VERSION = 1; + +/** Missing version metadata identifies a legacy Qtart runtime (version 0). */ +export function getQtartApiVersion(): number { + const version = (globalThis as any)._tart?.apiVersion; + return Number.isSafeInteger(version) && version >= 0 ? version : 0; +} + +function addQtartDatabaseCapabilities( + handle: WalletDbHandle, + temporaryStoragePath: string | undefined, + persistentStoragePath: string | undefined, +): WalletDbHandle { const exportPhysical = handle.exportToFile?.bind(handle); handle.exportToFile = async (directory, stem, forceFormat) => { if (forceFormat === "json") { @@ -89,9 +107,9 @@ function addQtartDatabaseCapabilities(handle: WalletDbHandle): WalletDbHandle { } return await exportPhysical(directory, stem, "sqlite3"); }; - handle.readBackupJson = async (path) => { + handle.readBackupJson = async (backupPath) => { const errObj = { errno: undefined }; - const file = qjsStd.open(path, "r", errObj); + const file = qjsStd.open(backupPath, "r", errObj); if (!file) { throw Error(`could not open file (errno=${errObj.errno})`); } @@ -101,6 +119,79 @@ function addQtartDatabaseCapabilities(handle: WalletDbHandle): WalletDbHandle { file.close(); } }; + handle.readBackupFile = async (backupPath, options) => { + if (backupPath.endsWith(".json")) { + options?.cancellationToken?.throwIfCancelled(); + const dump = await handle.readBackupJson!(backupPath); + options?.cancellationToken?.throwIfCancelled(); + return dump; + } + const errObj = { errno: undefined }; + const file = qjsStd.open(backupPath, "r", errObj); + if (!file) { + throw Error(`could not open file (errno=${errObj.errno})`); + } + file.close(); + if (!backupPath.endsWith(".sqlite3")) { + throw Error("DB file import only supports .json and .sqlite3 files"); + } + const suffix = `taler-wallet-import-${encodeCrock(getRandomBytes(16))}`; + const persistentDirectory = persistentStoragePath + ? persistentStoragePath.slice( + 0, + persistentStoragePath.lastIndexOf("/") + 1, + ) || "." + : undefined; + const sourceDirectory = + backupPath.slice(0, backupPath.lastIndexOf("/") + 1) || "."; + const temporaryDirectoryRoots = [ + temporaryStoragePath, + persistentDirectory, + sourceDirectory, + ].filter( + (candidate, index, candidates): candidate is string => + candidate !== undefined && candidates.indexOf(candidate) === index, + ); + let temporaryDirectory: string | undefined; + let mkdirResult = -1; + for (const root of temporaryDirectoryRoots) { + const separator = root.endsWith("/") ? "" : "/"; + const candidate = `${root}${separator}${suffix}`; + mkdirResult = qjsOs.mkdir(candidate, 0o700); + if (mkdirResult === 0) { + temporaryDirectory = candidate; + break; + } + } + if (!temporaryDirectory) { + throw Error( + `could not create temporary directory (errno=${-mkdirResult})`, + ); + } + const snapshotPath = `${temporaryDirectory}/snapshot.sqlite3`; + let sourceHasSidecar = false; + for (const suffix of ["-journal", "-wal"]) { + const [, statError] = qjsOs.stat(`${backupPath}${suffix}`); + if (statError === 0) { + sourceHasSidecar = true; + } + } + try { + return await readWalletDbSqliteFile( + await createQtartSqlite3Impl(), + backupPath, + snapshotPath, + options, + !sourceHasSidecar, + ); + } finally { + qjsOs.remove(`${snapshotPath}-journal`); + qjsOs.remove(`${snapshotPath}-shm`); + qjsOs.remove(`${snapshotPath}-wal`); + qjsOs.remove(snapshotPath); + qjsOs.remove(temporaryDirectory); + } + }; handle.importAnyDatabase = async (dump, finalize, options) => await importWalletDbDump( await createQtartSqlite3Impl(), @@ -115,11 +206,20 @@ function addQtartDatabaseCapabilities(handle: WalletDbHandle): WalletDbHandle { export async function createQtartSqlite3Impl(): Promise<Sqlite3Interface> { const tart: any = (globalThis as any)._tart; if (!tart) { - throw Error("globalThis._qtart not defined"); + throw Error("globalThis._tart not defined"); + } + const apiVersion = getQtartApiVersion(); + if (apiVersion < REQUIRED_QTART_API_VERSION) { + throw Error( + `Qtart API version ${REQUIRED_QTART_API_VERSION} or newer is required; runtime reports version ${apiVersion}`, + ); } return { - async open(filename: string) { - const internalDbHandle = tart.sqlite3Open(filename); + async open( + filename: string, + options: { readonly?: boolean; immutable?: boolean } = {}, + ) { + const internalDbHandle = tart.sqlite3Open(filename, options); let open = true; let closePromise: Promise<void> | undefined; const statements = new Set<Sqlite3Statement>(); @@ -228,6 +328,8 @@ async function makeSqliteDb( await dropExpiredMigrationBackup(db); return addQtartDatabaseCapabilities( new SqliteWalletDbHandle(await openNativeSqliteWalletDb(db)), + args.temporaryStoragePath, + filename, ); } @@ -262,16 +364,28 @@ async function makeSqliteDb( handle.migrateToNative = async (options) => { const result = await migrateWalletDbToNative(db, handle, options); connectionTransferred = true; - return addQtartDatabaseCapabilities(result.handle); + return addQtartDatabaseCapabilities( + result.handle, + args.temporaryStoragePath, + filename, + ); }; if (kind === "empty") { handle.openNativeIfEmpty = async () => { const result = await openNativeWalletDbForEmptyStorage(db); connectionTransferred = true; - return addQtartDatabaseCapabilities(result); + return addQtartDatabaseCapabilities( + result, + args.temporaryStoragePath, + filename, + ); }; } - return addQtartDatabaseCapabilities(handle); + return addQtartDatabaseCapabilities( + handle, + args.temporaryStoragePath, + filename, + ); } export async function createNativeWalletHost2( diff --git a/packages/taler-wallet-core/src/requests.test.ts b/packages/taler-wallet-core/src/requests.test.ts @@ -18,6 +18,7 @@ import assert from "node:assert"; import { test } from "node:test"; import { + CancellationToken, CoinStatus, ExchangeEntryStatus, ExchangeEntrySource, @@ -331,6 +332,17 @@ test("importDb request converts a foreign backend dump", async () => { error instanceof TalerError && error.errorDetail.code === TalerErrorCode.WALLET_CORE_API_BAD_REQUEST, ); + target.readBackupJson = async () => { + CancellationToken.CANCELLED.throwIfCancelled(); + }; + await assert.rejects( + wallet.client.call(WalletApiOperation.ImportDbFromFile, { + path: "cancelled.json", + }), + (error: unknown) => + error instanceof TalerError && + error.errorDetail.code === TalerErrorCode.WALLET_CORE_REQUEST_CANCELLED, + ); } finally { await source.close(); if (initialized) { diff --git a/packages/taler-wallet-core/src/requests.ts b/packages/taler-wallet-core/src/requests.ts @@ -1988,23 +1988,51 @@ async function handleImportDbFromFile( WalletApiOperation.ImportDbFromFile, req.progressToken, async () => { - if (!req.path.endsWith(".json")) { + if (!req.path.endsWith(".json") && !req.path.endsWith(".sqlite3")) { throw TalerError.fromDetail( TalerErrorCode.WALLET_CORE_API_BAD_REQUEST, { parameter: "path" }, - "DB file import only supports .json files at the moment", + "DB file import only supports .json and .sqlite3 files", ); } const db = wex.ws.db; - const readBackupJson = db.readBackupJson; - if (!readBackupJson) { + const readBackupFile = db.readBackupFile; + const readBackupJson = req.path.endsWith(".json") + ? db.readBackupJson + : undefined; + if (!readBackupFile && !readBackupJson) { throw TalerError.fromDetail( TalerErrorCode.WALLET_DB_BACKEND_UNSUPPORTED, { backend: db.name }, `the ${db.name} backend cannot read a database dump from a file`, ); } - const dump = await readBackupJson.call(db, req.path); + let dump: unknown; + try { + if (readBackupFile) { + dump = await readBackupFile.call(db, req.path, { + cancellationToken: wex.cancellationToken, + progressToken: req.progressToken, + }); + } else { + wex.cancellationToken.throwIfCancelled(); + dump = await readBackupJson!.call(db, req.path); + wex.cancellationToken.throwIfCancelled(); + } + } catch (e) { + if (e instanceof CancellationToken.CancellationError) { + throw TalerError.fromDetail( + TalerErrorCode.WALLET_CORE_REQUEST_CANCELLED, + {}, + ); + } + const detail = getErrorDetailFromException(e); + throw TalerError.fromDetail( + TalerErrorCode.WALLET_CORE_API_BAD_REQUEST, + { parameter: "path" }, + `could not read wallet database backup: ${detail.hint}`, + ); + } return await importDbDump(wex, dump, req.progressToken); }, ); diff --git a/packages/taler-wallet-core/src/wallet-api-types.ts b/packages/taler-wallet-core/src/wallet-api-types.ts @@ -1281,7 +1281,7 @@ export type ExportDbToFileOp = { }; /** - * Export the database from a file. + * Import the database from a JSON or SQLite file. * * CAUTION: Overrides existing data. */ diff --git a/packages/taler-wallet-core/src/wallet-db-gate.test.ts b/packages/taler-wallet-core/src/wallet-db-gate.test.ts @@ -87,15 +87,22 @@ test("optional host capabilities follow the current database handle", async () = let current = oldHandle; const admitted = new AdmittedWalletDbHandle(() => current, gate); + assert.strictEqual(admitted.readBackupFile, undefined); assert.strictEqual(admitted.readBackupJson, undefined); - newHandle.readBackupJson = async (path) => ({ path, backend: "sqlite" }); + newHandle.readBackupJson = async (path) => ({ path, backend: "legacy" }); + newHandle.readBackupFile = async (path) => ({ path, backend: "sqlite" }); current = newHandle; assert.deepStrictEqual(await admitted.readBackupJson!("wallet.json"), { path: "wallet.json", + backend: "legacy", + }); + assert.deepStrictEqual(await admitted.readBackupFile!("wallet.json"), { + path: "wallet.json", backend: "sqlite", }); current = oldHandle; + assert.strictEqual(admitted.readBackupFile, undefined); assert.strictEqual(admitted.readBackupJson, undefined); }); diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts @@ -266,6 +266,18 @@ export class AdmittedWalletDbHandle implements WalletDbHandle { }); } + get readBackupFile(): WalletDbHandle["readBackupFile"] | undefined { + if (!this.current().readBackupFile) return undefined; + return (...args) => + this.gate.runShared(async () => { + const db = this.current(); + const fn = db.readBackupFile; + if (!fn) + throw Error("current database backend cannot read this backup"); + return await fn.apply(db, args); + }); + } + get readBackupJson(): WalletDbHandle["readBackupJson"] | undefined { if (!this.current().readBackupJson) return undefined; return (...args) =>