commit 88bfeb5aa067156ded296874b0632f9ecbdeb3e1 parent 7b77fb788bf5b3f1e98fd245c823465edeaea932 Author: Florian Dold <dold@taler.net> Date: Fri, 28 Aug 2026 01:14:36 +0200 wallet-core: speed coin selection and indexed lookups Diffstat:
13 files changed, 650 insertions(+), 18 deletions(-)
diff --git a/packages/taler-harness/src/index.ts b/packages/taler-harness/src/index.ts @@ -333,6 +333,12 @@ advancedCli .maybeOption("numExchanges", ["--num-exchanges"], clk.INT, { help: "Exchanges to spread those over (default 3)", }) + .maybeOption("numTransactions", ["--num-transactions"], clk.INT, { + help: "Historical transactions to insert (default 5000)", + }) + .maybeOption("numWithdrawalGroups", ["--num-withdrawals"], clk.INT, { + help: "Historical withdrawal groups to insert (default 5000)", + }) .maybeOption("repeats", ["--repeats"], clk.INT, { help: "Runs per query; the median is reported (default 5)", }) @@ -347,6 +353,8 @@ advancedCli numCoins: args.benchWalletDb.numCoins, numDenominations: args.benchWalletDb.numDenominations, numExchanges: args.benchWalletDb.numExchanges, + numTransactions: args.benchWalletDb.numTransactions, + numWithdrawalGroups: args.benchWalletDb.numWithdrawalGroups, repeats: args.benchWalletDb.repeats, backends: args.benchWalletDb.backend ? [args.benchWalletDb.backend] diff --git a/packages/taler-wallet-core/src/coinSelection.ts b/packages/taler-wallet-core/src/coinSelection.ts @@ -1673,6 +1673,12 @@ function reduceSelectionFees( const smallGroup = groups[smallIndex]; const smallValue = getCandidateSelectionMeta(smallGroup[0]).value; const selectedSmallCoins = selectedCoinsByValue.get(smallValue) ?? []; + // A replacement always consumes at least two smaller coins. Avoid the + // quadratic walk over every larger denomination when this value was not + // selected, or was selected only once. + if (selectedSmallCoins.length < 2) { + continue; + } selectedSmallCoins.sort((a, b) => compareCandidateDenoms(b.denom, a.denom)); for ( let largeIndex = smallIndex + 1; @@ -1680,6 +1686,12 @@ function reduceSelectionFees( largeIndex++ ) { const largeGroup = groups[largeIndex]; + const largeValue = getCandidateSelectionMeta(largeGroup[0]).value; + // Groups are ordered by value. Once even all selected small coins + // cannot equal the next value, no later denomination can replace them. + if (largeValue > smallValue * BigInt(selectedSmallCoins.length)) { + break; + } const factor = denominationFactor(largeGroup[0], smallGroup[0], true); if (factor === undefined || factor < 2) { continue; @@ -1712,7 +1724,6 @@ function reduceSelectionFees( ) { replacementMade = true; selectedSmallCoins.splice(0, factor); - const largeValue = getCandidateSelectionMeta(largeDenom).value; const selectedLargeCoins = selectedCoinsByValue.get(largeValue) ?? []; selectedLargeCoins.push(replacement); diff --git a/packages/taler-wallet-core/src/db/indexeddb/schema.ts b/packages/taler-wallet-core/src/db/indexeddb/schema.ts @@ -169,7 +169,7 @@ export const TALER_WALLET_DB_GENERATION_PREFIX = `${TALER_WALLET_MAIN_DB_NAME}-g * backwards-compatible way or object stores and indices * are added. */ -export const WALLET_DB_MINOR_VERSION = 33; +export const WALLET_DB_MINOR_VERSION = 34; // FIXME: Should these be numeric codes? export type KycUserType = "individual" | "business"; @@ -1028,6 +1028,9 @@ export const WalletIndexedDbStoresV1 = { "byTalerWithdrawUri", "wgInfo.bankInfo.talerWithdrawUri", ), + byReservePub: describeIndex("byReservePub", "reservePub", { + versionAdded: 34, + }), }, ), planchets: describeStore( diff --git a/packages/taler-wallet-core/src/db/indexeddb/transaction.ts b/packages/taler-wallet-core/src/db/indexeddb/transaction.ts @@ -108,6 +108,10 @@ function getActiveKeyRange() { ); } +function denominationRefMapKey(ref: WalletDenomRef): string { + return `${ref.exchangeMasterPub}:${ref.denomPubHash}`; +} + export class IdbWalletTransaction implements WalletDbTransaction { tx: WalletIndexedDbTransaction; constructor(tx: WalletIndexedDbTransaction) { @@ -1299,6 +1303,19 @@ export class IdbWalletTransaction implements WalletDbTransaction { ); } + async getWithdrawalGroupsByReservePubs( + reservePubs: string[], + ): Promise<WalletWithdrawalGroup[]> { + const tx = this.tx; + const uniqueReservePubs = [...new Set(reservePubs)]; + const groups = await Promise.all( + uniqueReservePubs.map((reservePub) => + tx.withdrawalGroups.indexes.byReservePub.getAll(reservePub), + ), + ); + return groups.flat(); + } + async getWithdrawalGroupsByExchange( exchangeBaseUrl: string, ): Promise<WalletWithdrawalGroup[]> { @@ -1898,14 +1915,63 @@ export class IdbWalletTransaction implements WalletDbTransaction { async getDenominationsByRefs( refs: WalletDenomRef[], ): Promise<WalletDenomination[]> { - const records = await Promise.all( - refs.map((ref) => - this.tx.denominationsV2.get([ref.exchangeMasterPub, ref.denomPubHash]), - ), - ); - return records.filter( - (record): record is WalletDenomination => record !== undefined, + const refsByMaster = new Map<string, Map<string, WalletDenomRef>>(); + for (const ref of refs) { + let byKey = refsByMaster.get(ref.exchangeMasterPub); + if (!byKey) { + byKey = new Map(); + refsByMaster.set(ref.exchangeMasterPub, byKey); + } + byKey.set(denominationRefMapKey(ref), ref); + } + + const recordsByRef = new Map<string, WalletDenomination>(); + await Promise.all( + [...refsByMaster.entries()].map(async ([masterPub, byKey]) => { + const wanted = [...byKey.values()]; + const index = this.tx.denominationsV2.indexes.byExchangeMasterPub; + + // A large candidate set usually covers most current denominations of + // one master key. One index range read is then much cheaper on the + // SQLite-backed IndexedDB bridge than one request per compound key. + // Count first so a small subset of a very long historical key set + // keeps using point lookups instead of materialising unrelated rows. + if (wanted.length >= 8) { + const totalForMaster = await index.count(masterPub); + if (totalForMaster <= wanted.length * 4) { + const records = await index.getAll(masterPub); + for (const record of records) { + const key = denominationRefMapKey(record); + if (byKey.has(key)) { + recordsByRef.set(key, record); + } + } + return; + } + } + + const records = await Promise.all( + wanted.map((ref) => + this.tx.denominationsV2.get([ + ref.exchangeMasterPub, + ref.denomPubHash, + ]), + ), + ); + for (const record of records) { + if (record) { + recordsByRef.set(denominationRefMapKey(record), record); + } + } + }), ); + + // Preserve the DAL contract: missing records are skipped, while duplicate + // references retain their input positions. + return refs.flatMap((ref) => { + const record = recordsByRef.get(denominationRefMapKey(ref)); + return record ? [record] : []; + }); } async findDenominationByFamilyFromExpiry( diff --git a/packages/taler-wallet-core/src/db/sqlite/schema-migrations.test.ts b/packages/taler-wallet-core/src/db/sqlite/schema-migrations.test.ts @@ -225,6 +225,32 @@ test("visible availability migration adds the partial balance index", async () = } }); +test("withdrawal reserve lookup migration adds its index", async () => { + const { path, cleanup } = withTempDb(); + try { + let db = await openRaw(path); + await initSqliteWalletDb( + db, + schemaMigrations.filter((x) => x.version < 12), + ); + let indexes = await queryAll(db, "PRAGMA index_list(withdrawal_groups)"); + assert.ok( + !indexes.some((x) => x.name === "withdrawal_groups_by_reserve_pub"), + ); + await db.close(); + + db = await openRaw(path); + await initSqliteWalletDb(db); + indexes = await queryAll(db, "PRAGMA index_list(withdrawal_groups)"); + assert.ok( + indexes.some((x) => x.name === "withdrawal_groups_by_reserve_pub"), + ); + await db.close(); + } finally { + cleanup(); + } +}); + test("peer capability migration deterministically removes legacy duplicates", async () => { const { path, cleanup } = withTempDb(); try { diff --git a/packages/taler-wallet-core/src/db/sqlite/schema.ts b/packages/taler-wallet-core/src/db/sqlite/schema.ts @@ -85,7 +85,7 @@ * * Bump this when adding a migration to {@link schemaMigrations}. */ -export const SQLITE_SCHEMA_VERSION = 11; +export const SQLITE_SCHEMA_VERSION = 12; /** * Tables of the IndexedDB emulation, children before parents. @@ -1403,6 +1403,13 @@ export const schemaMigrations: SchemaMigration[] = [ "CREATE INDEX coin_availability_by_visible_count ON coin_availability (visible_coin_count) WHERE visible_coin_count > 0", ], }, + { + version: 12, + name: "withdrawal-group-reserve-public-key", + statements: [ + "CREATE INDEX withdrawal_groups_by_reserve_pub ON withdrawal_groups (reserve_pub)", + ], + }, ]; /** Native tables that contain wallet records (not schema bookkeeping). */ diff --git a/packages/taler-wallet-core/src/db/sqlite/transaction.ts b/packages/taler-wallet-core/src/db/sqlite/transaction.ts @@ -2564,6 +2564,32 @@ export class SqliteWalletTransaction implements WalletDbTransaction { return row ? this.rowToWithdrawalGroup(row) : undefined; } + async getWithdrawalGroupsByReservePubs( + reservePubs: string[], + ): Promise<WalletWithdrawalGroup[]> { + const unique = new Map<string, Uint8Array>(); + for (const reservePub of reservePubs) { + const blob = crockToDb(reservePub); + unique.set(blobKey(blob), blob); + } + const blobs = [...unique.values()]; + const groups: WalletWithdrawalGroup[] = []; + for (let offset = 0; offset < blobs.length; offset += 500) { + const chunk = blobs.slice(offset, offset + 500); + const params: Record<string, Uint8Array> = {}; + const placeholders = chunk.map((blob, i) => { + params[`p${i}`] = blob; + return `$p${i}`; + }); + const rows = await this.all( + `SELECT * FROM withdrawal_groups WHERE reserve_pub IN (${placeholders.join(", ")})`, + params, + ); + groups.push(...rows.map((row) => this.rowToWithdrawalGroup(row))); + } + return groups; + } + async getWithdrawalGroupsByExchange( exchangeBaseUrl: string, ): Promise<WalletWithdrawalGroup[]> { diff --git a/packages/taler-wallet-core/src/db/testing/benchmark.test.ts b/packages/taler-wallet-core/src/db/testing/benchmark.test.ts @@ -25,6 +25,8 @@ test("database benchmark smoke test returns real, comparable rows", async () => numCoins: 120, numDenominations: 12, numExchanges: 2, + numTransactions: 60, + numWithdrawalGroups: 60, repeats: 1, }; const results = []; diff --git a/packages/taler-wallet-core/src/db/testing/benchmark.ts b/packages/taler-wallet-core/src/db/testing/benchmark.ts @@ -32,21 +32,44 @@ */ import { + Amounts, AmountString, + Cache, CoinStatus, DenomKeyType, + Duration, encodeCrock, + ExchangeEntrySource, Logger, + TalerProtocolTimestamp, } from "@gnu-taler/taler-util"; import { CoinSourceType, DenominationVerificationStatus, + DbPreciseTimestamp, DbProtocolTimestamp, + ExchangeEntryDbRecordStatus, + ExchangeEntryDbUpdateStatus, WalletCoin, WalletCoinAvailability, WalletDenomination, + WalletExchangeDetails, + WalletExchangeEntry, + WalletTransactionMeta, + WalletWithdrawalGroup, + OPERATION_STATUS_DONE_FIRST, + OPERATION_STATUS_NONFINAL_FIRST, + WithdrawalGroupStatus, + WithdrawalRecordType, } from "../records.js"; import { getBalancesInsideTransaction } from "../../balance.js"; +import { + AvailableCoinsOfDenom, + PreviousPayCoins, + selectPayCoinsInTx, + selectPeerCoinsInTx, + testing_selectGreedy, +} from "../../coinSelection.js"; import { WalletExecutionContext } from "../../wallet.js"; import type { DbTxRunner } from "./conformance.js"; @@ -59,6 +82,10 @@ export interface DbBenchOptions { numDenominations: number; /** Exchanges to spread the denominations over. */ numExchanges: number; + /** Historical transaction metadata records to insert. */ + numTransactions: number; + /** Historical withdrawal groups to insert. */ + numWithdrawalGroups: number; /** How many times each query is repeated; the median is reported. */ repeats: number; } @@ -67,6 +94,8 @@ export const defaultDbBenchOptions: DbBenchOptions = { numCoins: 20000, numDenominations: 200, numExchanges: 3, + numTransactions: 5000, + numWithdrawalGroups: 5000, repeats: 5, }; @@ -130,10 +159,99 @@ function exchangeUrl(i: number): string { return `https://exchange-${i}.test/`; } +function advertisedFreshCoinsForDenom( + denomIndex: number, + opts: DbBenchOptions, +): number { + let fresh = 0; + for ( + let coinIndex = denomIndex; + coinIndex < opts.numCoins; + coinIndex += opts.numDenominations + ) { + if (Math.floor(coinIndex / opts.numDenominations) % 4 !== 0) { + fresh++; + } + } + return Math.min(10, fresh); +} + export async function populateDbBench( runner: DbTxRunner, opts: DbBenchOptions, ): Promise<void> { + await runner.runReadWriteTx(async (tx) => { + for (let exchange = 0; exchange < opts.numExchanges; exchange++) { + const baseUrl = exchangeUrl(exchange); + const masterPublicKey = key(`master-${exchange}`); + const details: WalletExchangeDetails = { + exchangeBaseUrl: baseUrl, + masterPublicKey, + currency: "TESTKUDOS", + auditors: [], + protocolVersionRange: "18:0:1", + tinyAmount: "TESTKUDOS:0.01" as AmountString, + reserveClosingDelay: { d_us: 1_000 }, + globalFees: [ + { + startDate: TalerProtocolTimestamp.zero(), + endDate: TalerProtocolTimestamp.never(), + historyFee: "TESTKUDOS:0" as AmountString, + accountFee: "TESTKUDOS:0" as AmountString, + purseFee: "TESTKUDOS:0.01" as AmountString, + historyTimeout: { d_us: 1_000 }, + purseTimeout: { d_us: 1_000 }, + purseLimit: 1_000, + signature: "benchmark-global-fee-signature", + }, + ], + wireInfo: { + accounts: [ + { + payto_uri: "payto://iban/DE76500202009817493529", + master_sig: "benchmark-wire-signature", + credit_restrictions: [], + debit_restrictions: [], + }, + ], + feesForType: { + iban: [ + { + wireFee: "TESTKUDOS:0.01" as AmountString, + closingFee: "TESTKUDOS:0" as AmountString, + startStamp: TalerProtocolTimestamp.zero(), + endStamp: TalerProtocolTimestamp.never(), + sig: "benchmark-wire-fee-signature", + }, + ], + }, + }, + bankComplianceLanguage: undefined, + defaultPeerPushExpiration: undefined, + }; + await tx.upsertExchangeDetails(details); + const entry: WalletExchangeEntry = { + baseUrl, + source: ExchangeEntrySource.Builtin, + detailsPointer: { + masterPublicKey, + currency: "TESTKUDOS", + updateClock: 1 as DbPreciseTimestamp, + }, + entryStatus: ExchangeEntryDbRecordStatus.Used, + updateStatus: ExchangeEntryDbUpdateStatus.Ready, + tosCurrentEtag: undefined, + tosAcceptedEtag: undefined, + tosAcceptedTimestamp: undefined, + lastUpdate: undefined, + nextUpdateStamp: 1_000 as DbPreciseTimestamp, + lastKeysEtag: undefined, + nextRefreshCheckStamp: 2_000 as DbPreciseTimestamp, + }; + await tx.upsertExchange(entry); + } + }); + // Denominations and their availability rows. await runner.runReadWriteTx(async (tx) => { const familySerials: number[] = []; @@ -157,6 +275,7 @@ export async function populateDbBench( const ex = exchangeUrl(exchange); const dph = hash(`denom-${d}`); const masterPub = key(`master-${exchange}`); + const freshCoinCount = advertisedFreshCoinsForDenom(d, opts); const denom: WalletDenomination = { denomPubHash: dph, denomPub: { @@ -192,12 +311,12 @@ export async function populateDbBench( maxAge: d % 2 === 0 ? 0 : 21, currency: "TESTKUDOS", value: "TESTKUDOS:1" as AmountString, - freshCoinCount: 10, - hasFreshCoins: 1, + freshCoinCount, + hasFreshCoins: freshCoinCount > 0 ? 1 : 0, // Most rows in a long-lived wallet describe denominations whose last // visible coin has already been spent. Keep that shape in the // benchmark instead of making every historical row contribute. - visibleCoinCount: d % 10 === 0 ? 10 : 0, + visibleCoinCount: d % 10 === 0 ? freshCoinCount : 0, }; await tx.upsertCoinAvailability(avail); } @@ -240,6 +359,63 @@ export async function populateDbBench( } }); } + + // Transaction metadata is the materialized index behind the transaction + // history screen. Keep a small active tail amid a long completed history, + // which is the shape that exercises both paging and task/balance filtering. + const transactionBatch = 1000; + for (let start = 0; start < opts.numTransactions; start += transactionBatch) { + await runner.runReadWriteTx(async (tx) => { + for ( + let i = start; + i < Math.min(start + transactionBatch, opts.numTransactions); + i++ + ) { + const rec: WalletTransactionMeta = { + transactionId: `txn:payment:bench-${i.toString().padStart(8, "0")}`, + // Repeated timestamps exercise the transactionId tie breaker in the + // composite paging index. + timestamp: (100_000 + Math.floor(i / 3)) as DbPreciseTimestamp, + status: + i < Math.min(5, opts.numTransactions) + ? OPERATION_STATUS_NONFINAL_FIRST + : OPERATION_STATUS_DONE_FIRST, + exchanges: [exchangeUrl(i % opts.numExchanges)], + currency: "TESTKUDOS", + }; + await tx.upsertTransactionMeta(rec); + } + }); + } + + // Withdrawal transfer-result references arrive from the bank as opaque + // strings containing the reserve public key. A long completed history is + // what distinguishes an indexed lookup from the former full-table scan. + const withdrawalBatch = 1000; + for ( + let start = 0; + start < opts.numWithdrawalGroups; + start += withdrawalBatch + ) { + await runner.runReadWriteTx(async (tx) => { + for ( + let i = start; + i < Math.min(start + withdrawalBatch, opts.numWithdrawalGroups); + i++ + ) { + const rec: WalletWithdrawalGroup = { + withdrawalGroupId: `bench-withdrawal-${i}`, + wgInfo: { withdrawalType: WithdrawalRecordType.BankManual }, + secretSeed: key(`withdrawal-seed-${i}`), + reservePub: key(`withdrawal-reserve-pub-${i}`), + reservePriv: key(`withdrawal-reserve-priv-${i}`), + timestampStart: (200_000 + i) as DbPreciseTimestamp, + status: WithdrawalGroupStatus.Done, + }; + await tx.upsertWithdrawalGroup(rec); + } + }); + } } /** @@ -382,6 +558,221 @@ export async function measureDbBenchQueries( ), ); + const selectionWex = { + ws: { + devExperimentState: {}, + denomInfoCache: new Cache( + Math.max(1_000, opts.numDenominations * 2), + Duration.fromSpec({ minutes: 1 }), + ), + config: { testing: { coinSelectionAlgorithm: "default" } }, + }, + } as WalletExecutionContext; + const payRequest = ( + amount: number, + extra: { + requiredMinimumAge?: number; + prevPayCoins?: PreviousPayCoins; + } = {}, + ) => ({ + restrictExchanges: undefined, + restrictWireMethod: "iban", + contractTermsAmount: Amounts.parseOrThrow(`TESTKUDOS:${amount}`), + depositFeeLimit: Amounts.parseOrThrow("TESTKUDOS:0"), + ...extra, + }); + const measureColdPay = async ( + amount: number, + extra: { + requiredMinimumAge?: number; + prevPayCoins?: PreviousPayCoins; + } = {}, + ): Promise<number> => { + selectionWex.ws.denomInfoCache.clear(); + return await runner.runReadWriteTx(async (tx) => { + const result = await selectPayCoinsInTx( + selectionWex, + tx, + payRequest(amount, extra), + ); + return result.type === "success" ? result.coinSel.coins.length : 0; + }); + }; + + await time("selectPayCoins cold (amount 10)", async () => measureColdPay(10)); + await time("selectPayCoins warm (amount 10)", async () => + runner.runReadWriteTx(async (tx) => { + const result = await selectPayCoinsInTx(selectionWex, tx, payRequest(10)); + return result.type === "success" ? result.coinSel.coins.length : 0; + }), + ); + await time("selectPayCoins age restricted", async () => + measureColdPay(10, { requiredMinimumAge: 18 }), + ); + await time("selectPayCoins insufficient balance", async () => + measureColdPay(Math.max(1, opts.numDenominations * 20)), + ); + await time("selectPayCoins many inputs", async () => + measureColdPay(Math.max(1, opts.numDenominations * 5)), + ); + await time("selectPayCoins repair", async () => + measureColdPay(10, { + prevPayCoins: [ + { + coinPub: key("coin-0"), + contribution: Amounts.parseOrThrow("TESTKUDOS:1"), + }, + ], + }), + ); + await time("selectPeerCoins cold (amount 10)", async () => { + selectionWex.ws.denomInfoCache.clear(); + return await runner.runReadWriteTx(async (tx) => { + const result = await selectPeerCoinsInTx(selectionWex, tx, { + instructedAmount: Amounts.parseOrThrow("TESTKUDOS:10"), + }); + return result.type === "success" ? result.result.coins.length : 0; + }); + }); + + // Isolate the CPU side of DD91 with distinct values. This deliberately + // defeats value grouping and used to make the fee-reduction pass compare + // every denomination pair, even when only one coin of a value was selected. + const algorithmCandidateCount = Math.min(opts.numDenominations, 5_000); + const algorithmCandidates: AvailableCoinsOfDenom[] = Array.from( + { length: algorithmCandidateCount }, + (_, i) => ({ + denomPub: { + cipher: DenomKeyType.Rsa, + rsa_public_key: `algorithm-rsa-${i}`, + age_mask: 0, + }, + denomPubHash: hash(`algorithm-denom-${i}`), + feeDeposit: "TESTKUDOS:0.01" as AmountString, + feeRefresh: "TESTKUDOS:0" as AmountString, + feeRefund: "TESTKUDOS:0" as AmountString, + feeWithdraw: "TESTKUDOS:0" as AmountString, + exchangeBaseUrl: exchangeUrl(0), + exchangeMasterPub: key("master-0"), + maxAge: 0, + numAvailable: 1, + stampExpireDeposit: TalerProtocolTimestamp.never(), + stampExpireLegal: TalerProtocolTimestamp.never(), + stampExpireWithdraw: TalerProtocolTimestamp.never(), + stampStart: TalerProtocolTimestamp.zero(), + value: `TESTKUDOS:${i + 1}` as AmountString, + isLost: false, + isOffered: true, + masterSig: hash(`algorithm-master-sig-${i}`), + }), + ); + const halfCandidateCount = Math.floor(algorithmCandidateCount / 2); + const algorithmAmount = Math.max( + 1, + Math.floor(((halfCandidateCount * (halfCandidateCount + 1)) / 2) * 0.8), + ); + await time( + `selectGreedy distinct values (${algorithmCandidateCount})`, + async () => { + const zero = Amounts.parseOrThrow("TESTKUDOS:0"); + const selected = testing_selectGreedy( + { wireFeesPerExchange: {}, algorithm: "default" }, + algorithmCandidates, + { + amountPayRemaining: Amounts.parseOrThrow( + `TESTKUDOS:${algorithmAmount}`, + ), + amountDepositFeeLimitRemaining: zero, + customerDepositFees: zero, + totalDepositFees: zero, + customerWireFees: zero, + wireFeeCoveredForExchange: new Set(), + }, + ); + return Object.values(selected ?? {}).reduce( + (sum, selection) => sum + selection.contributions.length, + 0, + ); + }, + ); + + const middleTransaction = Math.floor(opts.numTransactions / 2); + const middleTransactionId = `txn:payment:bench-${middleTransaction + .toString() + .padStart(8, "0")}`; + await time("getTransactionMeta (point lookup)", async () => + runner.runReadWriteTx(async (tx) => + (await tx.getTransactionMeta(middleTransactionId)) ? 1 : 0, + ), + ); + + await time("listTransactionMetaPage (first 50)", async () => + runner.runReadWriteTx( + async (tx) => + ( + await tx.listTransactionMetaPage({ + direction: "forward", + limit: 50, + }) + ).length, + ), + ); + + await time("listTransactionMetaPage (middle 50)", async () => + runner.runReadWriteTx( + async (tx) => + ( + await tx.listTransactionMetaPage({ + cursor: { + timestamp: (100_000 + + Math.floor(middleTransaction / 3)) as DbPreciseTimestamp, + transactionId: middleTransactionId, + }, + direction: "forward", + limit: 50, + }) + ).length, + ), + ); + + await time("listTransactionMetaPage (last 50)", async () => + runner.runReadWriteTx( + async (tx) => + ( + await tx.listTransactionMetaPage({ + direction: "backward", + limit: 50, + }) + ).length, + ), + ); + + await time("listTransactionMetaByStatus (active)", async () => + runner.runReadWriteTx( + async (tx) => + (await tx.listTransactionMetaByStatus({ onlyActive: true })).length, + ), + ); + + await time("listTransactionMetaByStatus (full history)", async () => + runner.runReadWriteTx( + async (tx) => + (await tx.listTransactionMetaByStatus({ onlyActive: false })).length, + ), + ); + + const middleWithdrawal = Math.floor(opts.numWithdrawalGroups / 2); + await time("getWithdrawalGroupsByReservePubs (reference)", async () => + runner.runReadWriteTx( + async (tx) => + ( + await tx.getWithdrawalGroupsByReservePubs([ + key(`withdrawal-reserve-pub-${middleWithdrawal}`), + ]) + ).length, + ), + ); + // A write-heavy transaction, to keep an eye on commit cost. await time("upsertCoin x100 (one tx)", async () => runner.runReadWriteTx(async (tx) => { @@ -428,6 +819,27 @@ export async function benchmarkOneBackend( `database holds ${actualCoins}. Refusing to report timings.`, ); } + const actualTransactions = await runner.runReadWriteTx( + async (tx) => + (await tx.listTransactionMetaByStatus({ onlyActive: false })).length, + ); + if (actualTransactions !== opts.numTransactions) { + throw Error( + `benchmark population is wrong: asked for ${opts.numTransactions} ` + + `transactions, database holds ${actualTransactions}. ` + + "Refusing to report timings.", + ); + } + const actualWithdrawalGroups = await runner.runReadWriteTx( + async (tx) => (await tx.listAllWithdrawalGroups()).length, + ); + if (actualWithdrawalGroups !== opts.numWithdrawalGroups) { + throw Error( + `benchmark population is wrong: asked for ${opts.numWithdrawalGroups} ` + + `withdrawal groups, database holds ${actualWithdrawalGroups}. ` + + "Refusing to report timings.", + ); + } const queries = await measureDbBenchQueries(runner, opts); return { backend: runner.name, @@ -450,6 +862,8 @@ export function formatDbBenchResults(results: DbBenchResult[]): string { `wallet DB benchmark: ${base.options.numCoins} coins, ` + `${base.options.numDenominations} denominations, ` + `${base.options.numExchanges} exchanges, ` + + `${base.options.numTransactions} transactions, ` + + `${base.options.numWithdrawalGroups} withdrawals, ` + `median of ${base.options.repeats}`, ); lines.push(""); diff --git a/packages/taler-wallet-core/src/db/testing/conformance-cases.ts b/packages/taler-wallet-core/src/db/testing/conformance-cases.ts @@ -2950,6 +2950,30 @@ export const conformanceCases: ConformanceCase[] = [ }, { + name: "withdrawal group: batch lookup by reserve public key", + async run(t, runner) { + const first = makeWithdrawalGroup("wg-reserve-first"); + const second = makeWithdrawalGroup("wg-reserve-second"); + await runner.runReadWriteTx(async (tx) => { + await tx.upsertWithdrawalGroup(first); + await tx.upsertWithdrawalGroup(second); + }); + const groups = await runner.runReadWriteTx((tx) => + tx.getWithdrawalGroupsByReservePubs([ + second.reservePub, + ck("missing-reserve"), + first.reservePub, + second.reservePub, + ]), + ); + t.deepEqual( + groups.map((group) => group.withdrawalGroupId).sort(), + [first.withdrawalGroupId, second.withdrawalGroupId].sort(), + ); + }, + }, + + { name: "withdrawal group: delete", async run(t, runner) { await runner.runReadWriteTx((tx) => diff --git a/packages/taler-wallet-core/src/db/transaction.ts b/packages/taler-wallet-core/src/db/transaction.ts @@ -871,6 +871,14 @@ export interface WalletDbTransaction { talerWithdrawUri: string, ): Promise<WalletWithdrawalGroup | undefined>; + /** + * Get the withdrawal groups whose reserve public keys occur in the input. + * Missing and duplicate keys do not add entries to the result. + */ + getWithdrawalGroupsByReservePubs( + reservePubs: string[], + ): Promise<WalletWithdrawalGroup[]>; + /** Get the withdrawal groups against an exchange. */ getWithdrawalGroupsByExchange( exchangeBaseUrl: string, diff --git a/packages/taler-wallet-core/src/transactions.test.ts b/packages/taler-wallet-core/src/transactions.test.ts @@ -18,6 +18,7 @@ import assert from "node:assert"; import { test } from "node:test"; import { constructTransactionIdentifier, + extractReservePublicKeyCandidates, ParsedTransactionIdentifier, parseTransactionIdentifier, resolveTransactionReference, @@ -42,11 +43,17 @@ const allIdentifiers: ParsedTransactionIdentifier[] = [ ]; function withdrawalReferenceContext( - groups: Array<Pick<WalletWithdrawalGroup, "withdrawalGroupId" | "reservePub">>, + groups: Array< + Pick<WalletWithdrawalGroup, "withdrawalGroupId" | "reservePub"> + >, ): WalletExecutionContext { const tx = { - async listAllWithdrawalGroups(): Promise<WalletWithdrawalGroup[]> { - return groups as WalletWithdrawalGroup[]; + async getWithdrawalGroupsByReservePubs( + reservePubs: string[], + ): Promise<WalletWithdrawalGroup[]> { + return groups.filter((group) => + reservePubs.includes(group.reservePub), + ) as WalletWithdrawalGroup[]; }, } as WalletDbTransaction; return { @@ -101,6 +108,12 @@ test("a bank withdrawal reference resolves by its embedded reserve public key", ); }); +test("reserve public key candidates include keys next to Crockford characters", () => { + const reservePub = "A".repeat(52); + const ref = `prefix-B${reservePub}C-suffix`; + assert.ok(extractReservePublicKeyCandidates(ref).includes(reservePub)); +}); + test("an unknown or ambiguous bank withdrawal reference is not resolved", async () => { const reservePubA = "A".repeat(52); const reservePubB = "B".repeat(52); diff --git a/packages/taler-wallet-core/src/transactions.ts b/packages/taler-wallet-core/src/transactions.ts @@ -265,6 +265,29 @@ function withLocalTransactionIdentifier( } as Transaction; } +const reservePublicKeyLength = 52; +const crockfordRunPattern = /[0-9A-HJKMNP-TV-Z]+/g; + +/** + * Find every possible reserve public key embedded in an opaque bank reference. + * Sliding over Crockford runs matters because the bank is free to put other + * Crockford characters immediately before or after the key. + */ +export function extractReservePublicKeyCandidates(ref: string): string[] { + const candidates = new Set<string>(); + for (const match of ref.matchAll(crockfordRunPattern)) { + const run = match[0]; + for ( + let offset = 0; + offset + reservePublicKeyLength <= run.length; + offset++ + ) { + candidates.add(run.slice(offset, offset + reservePublicKeyLength)); + } + } + return [...candidates]; +} + /** * Resolve either the stable transaction ID or the backend's optional local * identifier. A local ID is deliberately scoped by its type and never @@ -301,9 +324,10 @@ export async function resolveTransactionReference( // strings that contain the reserve public key. The status carried next to // the reference is deliberately not applied here: it is only a UI hint and // wallet-core's transaction state remains authoritative. + const reservePubCandidates = extractReservePublicKeyCandidates(ref); const withdrawalMatches = await wex.runWalletDbTx(async (tx) => - (await tx.listAllWithdrawalGroups()).filter((wg) => - ref.includes(wg.reservePub), + (await tx.getWithdrawalGroupsByReservePubs(reservePubCandidates)).filter( + (wg) => ref.includes(wg.reservePub), ), ); if (withdrawalMatches.length !== 1) {