From bd70ccfddfb9f993a5951a31be5bdc982fe1a58f Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Mon, 25 Mar 2024 20:31:56 +0100 Subject: wallet-core: re-denomination of withdrawal groups --- .../integrationtests/test-timetravel-withdraw.ts | 13 +- packages/taler-util/src/amounts.ts | 50 ++++- packages/taler-util/src/transactions-types.ts | 8 +- packages/taler-util/src/wallet-types.ts | 15 +- packages/taler-wallet-core/src/db.ts | 1 + packages/taler-wallet-core/src/denomSelection.ts | 24 ++- packages/taler-wallet-core/src/transactions.ts | 13 +- packages/taler-wallet-core/src/withdraw.ts | 208 ++++++++++++++++++++- 8 files changed, 300 insertions(+), 32 deletions(-) (limited to 'packages') diff --git a/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts b/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts index e594d2d72..9cd0beb42 100644 --- a/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts +++ b/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts @@ -21,6 +21,7 @@ import { Duration, TransactionMajorState, TransactionType, + j2s, } from "@gnu-taler/taler-util"; import { WalletApiOperation } from "@gnu-taler/taler-wallet-core"; import { GlobalTestState } from "../harness/harness.js"; @@ -69,7 +70,7 @@ export async function runTimetravelWithdrawTest(t: GlobalTestState) { console.log("starting withdrawal via bank"); // This should fail, as the wallet didn't time travel yet. - await withdrawViaBankV2(t, { + const wres2 = await withdrawViaBankV2(t, { walletClient, bank, exchange, @@ -82,9 +83,11 @@ export async function runTimetravelWithdrawTest(t: GlobalTestState) { { const transactions = await walletClient.call( WalletApiOperation.GetTransactions, - {}, + { + sort: "stable-ascending", + }, ); - console.log(transactions); + console.log(j2s(transactions)); const types = transactions.transactions.map((x) => x.type); t.assertDeepEqual(types, ["withdrawal", "withdrawal"]); const wtrans = transactions.transactions[1]; @@ -98,9 +101,9 @@ export async function runTimetravelWithdrawTest(t: GlobalTestState) { offsetMs: Duration.toMilliseconds(timetravelDuration), }); - // This doesn't work yet, see https://bugs.taler.net/n/6585 + // The wallet should do denomination re-selection and succeed - // await wallet.runUntilDone({ maxRetries: 5 }); + await wres2.withdrawalFinishedCond; } runTimetravelWithdrawTest.suites = ["wallet"]; diff --git a/packages/taler-util/src/amounts.ts b/packages/taler-util/src/amounts.ts index 4b7063fd2..82a3d3b68 100644 --- a/packages/taler-util/src/amounts.ts +++ b/packages/taler-util/src/amounts.ts @@ -76,6 +76,48 @@ export interface AmountJson { readonly currency: string; } +/** + * Immutable amount. + */ +export class Amount { + static from(a: AmountLike): Amount { + return new Amount(Amounts.parseOrThrow(a), 0); + } + + static zeroOfCurrency(currency: string): Amount { + return new Amount(Amounts.zeroOfCurrency(currency), 0); + } + + add(...a: AmountLike[]): Amount { + if (this.saturated) { + return this; + } + const r = Amounts.add(this.val, ...a); + return new Amount(r.amount, r.saturated ? 1 : 0); + } + + mult(n: number): Amount { + if (this.saturated) { + return this; + } + const r = Amounts.mult(this, n); + return new Amount(r.amount, r.saturated ? 1 : 0); + } + + toJson(): AmountJson { + return { ...this.val }; + } + + toString(): AmountString { + return Amounts.stringify(this.val); + } + + private constructor( + private val: AmountJson, + private saturated: number, + ) {} +} + export const codecForAmountJson = (): Codec => buildCodecForObject() .property("currency", codecForString()) @@ -118,7 +160,7 @@ export interface Result { /** * Type for things that are treated like amounts. */ -export type AmountLike = string | AmountString | AmountJson; +export type AmountLike = string | AmountString | AmountJson | Amount; export interface DivmodResult { quotient: number; @@ -162,6 +204,9 @@ export class Amounts { if (typeof amt === "string") { return Amounts.parseOrThrow(amt); } + if (amt instanceof Amount) { + return amt.toJson(); + } return amt; } @@ -406,6 +451,9 @@ export class Amounts { * throw if the input is not a valid amount. */ static parseOrThrow(s: AmountLike): AmountJson { + if (s instanceof Amount) { + return s.toJson(); + } if (typeof s === "object") { if (typeof s.currency !== "string") { throw Error("invalid amount object"); diff --git a/packages/taler-util/src/transactions-types.ts b/packages/taler-util/src/transactions-types.ts index c5d838809..8c4c2c7ed 100644 --- a/packages/taler-util/src/transactions-types.ts +++ b/packages/taler-util/src/transactions-types.ts @@ -77,8 +77,13 @@ export interface TransactionsRequest { * Sort order of the transaction items. * By default, items are sorted ascending by their * main timestamp. + * + * ascending: ascending by timestamp, but pending transactions first + * descending: ascending by timestamp, but pending transactions first + * stable-ascending: ascending by timestamp, with pending transactions amidst other transactions + * (stable in the sense of: pending transactions don't jump around) */ - sort?: "ascending" | "descending"; + sort?: "ascending" | "descending" | "stable-ascending"; /** * If true, include all refreshes in the transactions list. @@ -747,6 +752,7 @@ export const codecForTransactionsRequest = (): Codec => codecForEither( codecForConstString("ascending"), codecForConstString("descending"), + codecForConstString("stable-ascending"), ), ), ) diff --git a/packages/taler-util/src/wallet-types.ts b/packages/taler-util/src/wallet-types.ts index 0b09b0dbf..723e5a282 100644 --- a/packages/taler-util/src/wallet-types.ts +++ b/packages/taler-util/src/wallet-types.ts @@ -1548,16 +1548,23 @@ export interface WithdrawalDetailsForAmount { scopeInfo: ScopeInfo; } +export interface DenomSelItem { + denomPubHash: string; + count: number; + /** + * Number of denoms/planchets to skip, because + * a re-denomination effectively deleted them. + */ + skip?: number; +} + /** * Selected denominations withn some extra info. */ export interface DenomSelectionState { totalCoinValue: AmountString; totalWithdrawCost: AmountString; - selectedDenoms: { - denomPubHash: string; - count: number; - }[]; + selectedDenoms: DenomSelItem[]; } /** diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts index b59efe034..8b7aede57 100644 --- a/packages/taler-wallet-core/src/db.ts +++ b/packages/taler-wallet-core/src/db.ts @@ -699,6 +699,7 @@ export enum PlanchetStatus { Pending = 0x0100_0000, KycRequired = 0x0100_0001, WithdrawalDone = 0x0500_000, + AbortedReplaced = 0x0503_0001, } /** diff --git a/packages/taler-wallet-core/src/denomSelection.ts b/packages/taler-wallet-core/src/denomSelection.ts index 12f8f8971..dd5ec60d8 100644 --- a/packages/taler-wallet-core/src/denomSelection.ts +++ b/packages/taler-wallet-core/src/denomSelection.ts @@ -58,6 +58,12 @@ export function selectWithdrawalDenominations( denoms = denoms.filter((d) => isWithdrawableDenom(d, denomselAllowLate)); denoms.sort((d1, d2) => Amounts.cmp(d2.value, d1.value)); + if (logger.shouldLogTrace()) { + logger.trace( + `selecting withdrawal denoms for ${Amounts.stringify(amountAvailable)}`, + ); + } + for (const d of denoms) { const cost = Amounts.add(d.value, d.fees.feeWithdraw).amount; const res = Amounts.divmod(remaining, cost); @@ -78,19 +84,23 @@ export function selectWithdrawalDenominations( }); } + if (logger.shouldLogTrace()) { + logger.trace( + `denom_pub_hash=${ + d.denomPubHash + }, count=${count}, val=${Amounts.stringify( + d.value, + )}, wdfee=${Amounts.stringify(d.fees.feeWithdraw)}`, + ); + } + if (Amounts.isZero(remaining)) { break; } } if (logger.shouldLogTrace()) { - logger.trace( - `selected withdrawal denoms for ${Amounts.stringify(totalCoinValue)}`, - ); - for (const sd of selectedDenoms) { - logger.trace(`denom_pub_hash=${sd.denomPubHash}, count=${sd.count}`); - } - logger.trace("(end of withdrawal denom list)"); + logger.trace("(end of denom selection)"); } return { diff --git a/packages/taler-wallet-core/src/transactions.ts b/packages/taler-wallet-core/src/transactions.ts index e2e188e74..0e3f4a3fb 100644 --- a/packages/taler-wallet-core/src/transactions.ts +++ b/packages/taler-wallet-core/src/transactions.ts @@ -132,7 +132,7 @@ import { computeTipTransactionActions, RewardTransactionContext, } from "./reward.js"; -import type { InternalWalletState, WalletExecutionContext } from "./wallet.js"; +import type { WalletExecutionContext } from "./wallet.js"; import { augmentPaytoUrisForWithdrawal, computeWithdrawalTransactionActions, @@ -1487,9 +1487,6 @@ export async function getTransactions( x.txState.major === TransactionMajorState.Aborting || x.txState.major === TransactionMajorState.Dialog; - const txPending = transactions.filter((x) => isPending(x)); - const txNotPending = transactions.filter((x) => !isPending(x)); - let sortSign: number; if (transactionsRequest?.sort == "descending") { sortSign = -1; @@ -1510,6 +1507,14 @@ export async function getTransactions( return sortSign * tsCmp; }; + if (transactionsRequest?.sort === "stable-ascending") { + transactions.sort(txCmp); + return { transactions }; + } + + const txPending = transactions.filter((x) => isPending(x)); + const txNotPending = transactions.filter((x) => !isPending(x)); + txPending.sort(txCmp); txNotPending.sort(txCmp); diff --git a/packages/taler-wallet-core/src/withdraw.ts b/packages/taler-wallet-core/src/withdraw.ts index 9132d2b09..625d5dca4 100644 --- a/packages/taler-wallet-core/src/withdraw.ts +++ b/packages/taler-wallet-core/src/withdraw.ts @@ -27,6 +27,7 @@ import { AcceptManualWithdrawalResult, AcceptWithdrawalResponse, AgeRestriction, + Amount, AmountJson, AmountLike, AmountString, @@ -37,6 +38,7 @@ import { CoinStatus, CurrencySpecification, DenomKeyType, + DenomSelItem, DenomSelectionState, Duration, ExchangeBatchWithdrawRequest, @@ -92,6 +94,7 @@ import { HttpResponse, readSuccessResponseJsonOrErrorCode, readSuccessResponseJsonOrThrow, + readTalerErrorResponse, throwUnexpectedRequestError, } from "@gnu-taler/taler-util/http"; import { @@ -665,15 +668,22 @@ async function processPlanchetGenerate( return; } let ci = 0; + let isSkipped = false; let maybeDenomPubHash: string | undefined; for (let di = 0; di < withdrawalGroup.denomsSel.selectedDenoms.length; di++) { const d = withdrawalGroup.denomsSel.selectedDenoms[di]; if (coinIdx >= ci && coinIdx < ci + d.count) { maybeDenomPubHash = d.denomPubHash; + if (coinIdx >= ci + d.count - (d.skip ?? 0)) { + isSkipped = true; + } break; } ci += d.count; } + if (isSkipped) { + return; + } if (!maybeDenomPubHash) { throw Error("invariant violated"); } @@ -938,6 +948,9 @@ async function processPlanchetExchangeBatchRequest( logger.warn("processPlanchet: planchet already withdrawn"); continue; } + if (planchet.planchetStatus === PlanchetStatus.AbortedReplaced) { + continue; + } const denom = await getDenomInfo( wex, tx, @@ -968,10 +981,11 @@ async function processPlanchetExchangeBatchRequest( }; } - async function storeCoinError(e: any, coinIdx: number): Promise { - const errDetail = getErrorDetailFromException(e); - logger.trace("withdrawal request failed", e); - logger.trace(String(e)); + async function storeCoinError( + errDetail: TalerErrorDetail, + coinIdx: number, + ): Promise { + logger.trace(`withdrawal request failed: ${j2s(errDetail)}`); await wex.db.runReadWriteTx(["planchets"], async (tx) => { let planchet = await tx.planchets.indexes.byGroupAndIndex.get([ withdrawalGroup.withdrawalGroupId, @@ -1006,6 +1020,15 @@ async function processPlanchetExchangeBatchRequest( coinIdxs: [], }; } + if (resp.status === HttpStatusCode.Gone) { + const e = await readTalerErrorResponse(resp); + // FIXME: Store in place of the planchet that is actually affected! + await storeCoinError(e, requestCoinIdxs[0]); + return { + batchResp: { ev_sigs: [] }, + coinIdxs: [], + }; + } const r = await readSuccessResponseJsonOrThrow( resp, codecForExchangeWithdrawBatchResponse(), @@ -1015,7 +1038,10 @@ async function processPlanchetExchangeBatchRequest( batchResp: r, }; } catch (e) { - await storeCoinError(e, requestCoinIdxs[0]); + const errDetail = getErrorDetailFromException(e); + // We don't know which coin is affected, so we store the error + // with the first coin of the batch. + await storeCoinError(errDetail, requestCoinIdxs[0]); return { batchResp: { ev_sigs: [] }, coinIdxs: [], @@ -1488,6 +1514,128 @@ async function processWithdrawalGroupPendingKyc( return TaskRunResult.backoff(); } +async function redenominateWithdrawal( + wex: WalletExecutionContext, + withdrawalGroupId: string, +): Promise { + logger.trace(`redenominating withdrawal group ${withdrawalGroupId}`); + await wex.db.runReadWriteTx( + ["withdrawalGroups", "planchets", "denominations"], + async (tx) => { + const wg = await tx.withdrawalGroups.get(withdrawalGroupId); + if (!wg) { + return; + } + const currency = Amounts.currencyOf(wg.denomsSel.totalWithdrawCost); + const exchangeBaseUrl = wg.exchangeBaseUrl; + + const candidates = await getCandidateWithdrawalDenomsTx( + wex, + tx, + exchangeBaseUrl, + currency, + ); + + const oldSel = wg.denomsSel; + + if (logger.shouldLogTrace()) { + logger.trace(`old denom sel: ${j2s(oldSel)}`); + } + + let zero = Amount.zeroOfCurrency(currency); + let amountRemaining = zero; + let prevTotalCoinValue = zero; + let prevTotalWithdrawalCost = zero; + let prevDenoms: DenomSelItem[] = []; + let coinIndex = 0; + for (let i = 0; i < oldSel.selectedDenoms.length; i++) { + const sel = wg.denomsSel.selectedDenoms[i]; + const denom = await tx.denominations.get([ + exchangeBaseUrl, + sel.denomPubHash, + ]); + if (!denom) { + throw Error("denom in use but not not found"); + } + // FIXME: Also check planchet if there was a different error or planchet already withdrawn + let denomOkay = isWithdrawableDenom( + denom, + wex.ws.config.testing.denomselAllowLate, + ); + const numCoins = sel.count - (sel.skip ?? 0); + const denomValue = Amount.from(denom.value).mult(numCoins); + const denomFeeWithdraw = Amount.from(denom.fees.feeWithdraw).mult( + numCoins, + ); + if (denomOkay) { + prevTotalCoinValue = prevTotalCoinValue.add(denomValue); + prevTotalWithdrawalCost = prevTotalWithdrawalCost.add( + denomValue, + denomFeeWithdraw, + ); + prevDenoms.push({ + count: sel.count, + denomPubHash: sel.denomPubHash, + skip: sel.skip, + }); + } else { + amountRemaining = amountRemaining.add(denomValue, denomFeeWithdraw); + prevDenoms.push({ + count: sel.count, + denomPubHash: sel.denomPubHash, + skip: (sel.skip ?? 0) + numCoins, + }); + + for (let j = 0; j < sel.count; j++) { + const ci = coinIndex + j; + const p = await tx.planchets.indexes.byGroupAndIndex.get([ + withdrawalGroupId, + ci, + ]); + if (!p) { + // Maybe planchet wasn't yet generated. + // No problem! + logger.info( + `not aborting planchet #${coinIndex}, planchet not found`, + ); + continue; + } + logger.info(`aborting planchet #${coinIndex}`); + p.planchetStatus = PlanchetStatus.AbortedReplaced; + await tx.planchets.put(p); + } + } + + coinIndex += sel.count; + } + + const newSel = selectWithdrawalDenominations( + amountRemaining.toJson(), + candidates, + ); + + if (logger.shouldLogTrace()) { + logger.trace(`new denom sel: ${j2s(newSel)}`); + } + + const mergedSel: DenomSelectionState = { + selectedDenoms: [...prevDenoms, ...newSel.selectedDenoms], + totalCoinValue: zero + .add(prevTotalCoinValue, newSel.totalCoinValue) + .toString(), + totalWithdrawCost: zero + .add(prevTotalWithdrawalCost, newSel.totalWithdrawCost) + .toString(), + }; + wg.denomsSel = mergedSel; + if (logger.shouldLogTrace()) { + logger.trace(`merged denom sel: ${j2s(mergedSel)}`); + } + await tx.withdrawalGroups.put(wg); + }, + ); +} + async function processWithdrawalGroupPendingReady( wex: WalletExecutionContext, withdrawalGroup: WithdrawalGroupRecord, @@ -1498,6 +1646,8 @@ async function processWithdrawalGroupPendingReady( withdrawalGroupId, }); + const exchangeBaseUrl = withdrawalGroup.exchangeBaseUrl; + await fetchFreshExchange(wex, withdrawalGroup.exchangeBaseUrl); if (withdrawalGroup.denomsSel.selectedDenoms.length === 0) { @@ -1576,9 +1726,41 @@ async function processWithdrawalGroupPendingReady( await Promise.all(work); } - let numFinished = 0; + let redenomRequired = false; + + await wex.db.runReadOnlyTx(["planchets"], async (tx) => { + const planchets = + await tx.planchets.indexes.byGroup.getAll(withdrawalGroupId); + for (const p of planchets) { + if (p.planchetStatus !== PlanchetStatus.Pending) { + continue; + } + if (!p.lastError) { + continue; + } + switch (p.lastError.code) { + case TalerErrorCode.EXCHANGE_GENERIC_DENOMINATION_EXPIRED: + case TalerErrorCode.EXCHANGE_GENERIC_DENOMINATION_REVOKED: + redenomRequired = true; + return; + } + } + }); + + if (redenomRequired) { + logger.warn(`withdrawal ${withdrawalGroupId} requires redenomination`); + await fetchFreshExchange(wex, exchangeBaseUrl, { + forceUpdate: true, + }); + await updateWithdrawalDenoms(wex, exchangeBaseUrl); + await redenominateWithdrawal(wex, withdrawalGroupId); + return TaskRunResult.backoff(); + } + const errorsPerCoin: Record = {}; let numPlanchetErrors = 0; + let numActive = 0; + let numDone = 0; const maxReportedErrors = 5; const res = await wex.db.runReadWriteTx( @@ -1592,8 +1774,14 @@ async function processWithdrawalGroupPendingReady( await tx.planchets.indexes.byGroup .iter(withdrawalGroupId) .forEach((x) => { - if (x.planchetStatus === PlanchetStatus.WithdrawalDone) { - numFinished++; + switch (x.planchetStatus) { + case PlanchetStatus.KycRequired: + case PlanchetStatus.Pending: + numActive++; + break; + case PlanchetStatus.WithdrawalDone: + numDone++; + break; } if (x.lastError) { numPlanchetErrors++; @@ -1603,8 +1791,8 @@ async function processWithdrawalGroupPendingReady( } }); const oldTxState = computeWithdrawalTransactionStatus(wg); - logger.info(`now withdrawn ${numFinished} of ${numTotalCoins} coins`); - if (wg.timestampFinish === undefined && numFinished === numTotalCoins) { + logger.info(`now withdrawn ${numDone} of ${numTotalCoins} coins`); + if (wg.timestampFinish === undefined && numActive === 0) { wg.timestampFinish = timestampPreciseToDb(TalerPreciseTimestamp.now()); wg.status = WithdrawalGroupStatus.Done; await makeCoinsVisible(wex, tx, transactionId); -- cgit v1.2.3