wallet-core: re-denomination of withdrawal groups

8 files changed, 300 insertions, 32 deletions

diff --git a/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts b/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts
index e594d2d72..9cd0beb42 100644
--- a/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts
+++ b/packages/taler-harness/src/integrationtests/test-timetravel-withdraw.ts
@@ -21,6 +21,7 @@ import {
   Duration,
   TransactionMajorState,
   TransactionType,
+  j2s,
 } from "@gnu-taler/taler-util";
 import { WalletApiOperation } from "@gnu-taler/taler-wallet-core";
 import { GlobalTestState } from "../harness/harness.js";
@@ -69,7 +70,7 @@ export async function runTimetravelWithdrawTest(t: GlobalTestState) {
   console.log("starting withdrawal via bank");
 
   // This should fail, as the wallet didn't time travel yet.
-  await withdrawViaBankV2(t, {
+  const wres2 = await withdrawViaBankV2(t, {
     walletClient,
     bank,
     exchange,
@@ -82,9 +83,11 @@ export async function runTimetravelWithdrawTest(t: GlobalTestState) {
   {
     const transactions = await walletClient.call(
       WalletApiOperation.GetTransactions,
-      {},
+      {
+        sort: "stable-ascending",
+      },
     );
-    console.log(transactions);
+    console.log(j2s(transactions));
     const types = transactions.transactions.map((x) => x.type);
     t.assertDeepEqual(types, ["withdrawal", "withdrawal"]);
     const wtrans = transactions.transactions[1];
@@ -98,9 +101,9 @@ export async function runTimetravelWithdrawTest(t: GlobalTestState) {
     offsetMs: Duration.toMilliseconds(timetravelDuration),
   });
 
-  // This doesn't work yet, see https://bugs.taler.net/n/6585
+  // The wallet should do denomination re-selection and succeed
 
-  // await wallet.runUntilDone({ maxRetries: 5 });
+  await wres2.withdrawalFinishedCond;
 }
 
 runTimetravelWithdrawTest.suites = ["wallet"];
diff --git a/packages/taler-util/src/amounts.ts b/packages/taler-util/src/amounts.ts
index 4b7063fd2..82a3d3b68 100644
--- a/packages/taler-util/src/amounts.ts
+++ b/packages/taler-util/src/amounts.ts
@@ -76,6 +76,48 @@ export interface AmountJson {
   readonly currency: string;
 }
 
+/**
+ * Immutable amount.
+ */
+export class Amount {
+  static from(a: AmountLike): Amount {
+    return new Amount(Amounts.parseOrThrow(a), 0);
+  }
+
+  static zeroOfCurrency(currency: string): Amount {
+    return new Amount(Amounts.zeroOfCurrency(currency), 0);
+  }
+
+  add(...a: AmountLike[]): Amount {
+    if (this.saturated) {
+      return this;
+    }
+    const r = Amounts.add(this.val, ...a);
+    return new Amount(r.amount, r.saturated ? 1 : 0);
+  }
+
+  mult(n: number): Amount {
+    if (this.saturated) {
+      return this;
+    }
+    const r = Amounts.mult(this, n);
+    return new Amount(r.amount, r.saturated ? 1 : 0);
+  }
+
+  toJson(): AmountJson {
+    return { ...this.val };
+  }
+
+  toString(): AmountString {
+    return Amounts.stringify(this.val);
+  }
+
+  private constructor(
+    private val: AmountJson,
+    private saturated: number,
+  ) {}
+}
+
 export const codecForAmountJson = (): Codec<AmountJson> =>
   buildCodecForObject<AmountJson>()
     .property("currency", codecForString())
@@ -118,7 +160,7 @@ export interface Result {
 /**
  * Type for things that are treated like amounts.
  */
-export type AmountLike = string | AmountString | AmountJson;
+export type AmountLike = string | AmountString | AmountJson | Amount;
 
 export interface DivmodResult {
   quotient: number;
@@ -162,6 +204,9 @@ export class Amounts {
     if (typeof amt === "string") {
       return Amounts.parseOrThrow(amt);
     }
+    if (amt instanceof Amount) {
+      return amt.toJson();
+    }
     return amt;
   }
 
@@ -406,6 +451,9 @@ export class Amounts {
    * throw if the input is not a valid amount.
    */
   static parseOrThrow(s: AmountLike): AmountJson {
+    if (s instanceof Amount) {
+      return s.toJson();
+    }
     if (typeof s === "object") {
       if (typeof s.currency !== "string") {
         throw Error("invalid amount object");
diff --git a/packages/taler-util/src/transactions-types.ts b/packages/taler-util/src/transactions-types.ts
index c5d838809..8c4c2c7ed 100644
--- a/packages/taler-util/src/transactions-types.ts
+++ b/packages/taler-util/src/transactions-types.ts
@@ -77,8 +77,13 @@ export interface TransactionsRequest {
    * Sort order of the transaction items.
    * By default, items are sorted ascending by their
    * main timestamp.
+   *
+   * ascending: ascending by timestamp, but pending transactions first
+   * descending: ascending by timestamp, but pending transactions first
+   * stable-ascending: ascending by timestamp, with pending transactions amidst other transactions
+   *    (stable in the sense of: pending transactions don't jump around)
    */
-  sort?: "ascending" | "descending";
+  sort?: "ascending" | "descending" | "stable-ascending";
 
   /**
    * If true, include all refreshes in the transactions list.
@@ -747,6 +752,7 @@ export const codecForTransactionsRequest = (): Codec<TransactionsRequest> =>
         codecForEither(
           codecForConstString("ascending"),
           codecForConstString("descending"),
+          codecForConstString("stable-ascending"),
         ),
       ),
     )
diff --git a/packages/taler-util/src/wallet-types.ts b/packages/taler-util/src/wallet-types.ts
index 0b09b0dbf..723e5a282 100644
--- a/packages/taler-util/src/wallet-types.ts
+++ b/packages/taler-util/src/wallet-types.ts
@@ -1548,16 +1548,23 @@ export interface WithdrawalDetailsForAmount {
   scopeInfo: ScopeInfo;
 }
 
+export interface DenomSelItem {
+  denomPubHash: string;
+  count: number;
+  /**
+   * Number of denoms/planchets to skip, because
+   * a re-denomination effectively deleted them.
+   */
+  skip?: number;
+}
+
 /**
  * Selected denominations withn some extra info.
  */
 export interface DenomSelectionState {
   totalCoinValue: AmountString;
   totalWithdrawCost: AmountString;
-  selectedDenoms: {
-    denomPubHash: string;
-    count: number;
-  }[];
+  selectedDenoms: DenomSelItem[];
 }
 
 /**
diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts
index b59efe034..8b7aede57 100644
--- a/packages/taler-wallet-core/src/db.ts
+++ b/packages/taler-wallet-core/src/db.ts
@@ -699,6 +699,7 @@ export enum PlanchetStatus {
   Pending = 0x0100_0000,
   KycRequired = 0x0100_0001,
   WithdrawalDone = 0x0500_000,
+  AbortedReplaced = 0x0503_0001,
 }
 
 /**
diff --git a/packages/taler-wallet-core/src/denomSelection.ts b/packages/taler-wallet-core/src/denomSelection.ts
index 12f8f8971..dd5ec60d8 100644
--- a/packages/taler-wallet-core/src/denomSelection.ts
+++ b/packages/taler-wallet-core/src/denomSelection.ts
@@ -58,6 +58,12 @@ export function selectWithdrawalDenominations(
   denoms = denoms.filter((d) => isWithdrawableDenom(d, denomselAllowLate));
   denoms.sort((d1, d2) => Amounts.cmp(d2.value, d1.value));
 
+  if (logger.shouldLogTrace()) {
+    logger.trace(
+      `selecting withdrawal denoms for ${Amounts.stringify(amountAvailable)}`,
+    );
+  }
+
   for (const d of denoms) {
     const cost = Amounts.add(d.value, d.fees.feeWithdraw).amount;
     const res = Amounts.divmod(remaining, cost);
@@ -78,19 +84,23 @@ export function selectWithdrawalDenominations(
       });
     }
 
+    if (logger.shouldLogTrace()) {
+      logger.trace(
+        `denom_pub_hash=${
+          d.denomPubHash
+        }, count=${count}, val=${Amounts.stringify(
+          d.value,
+        )}, wdfee=${Amounts.stringify(d.fees.feeWithdraw)}`,
+      );
+    }
+
     if (Amounts.isZero(remaining)) {
       break;
     }
   }
 
   if (logger.shouldLogTrace()) {
-    logger.trace(
-      `selected withdrawal denoms for ${Amounts.stringify(totalCoinValue)}`,
-    );
-    for (const sd of selectedDenoms) {
-      logger.trace(`denom_pub_hash=${sd.denomPubHash}, count=${sd.count}`);
-    }
-    logger.trace("(end of withdrawal denom list)");
+    logger.trace("(end of denom selection)");
   }
 
   return {
diff --git a/packages/taler-wallet-core/src/transactions.ts b/packages/taler-wallet-core/src/transactions.ts
index e2e188e74..0e3f4a3fb 100644
--- a/packages/taler-wallet-core/src/transactions.ts
+++ b/packages/taler-wallet-core/src/transactions.ts
@@ -132,7 +132,7 @@ import {
   computeTipTransactionActions,
   RewardTransactionContext,
 } from "./reward.js";
-import type { InternalWalletState, WalletExecutionContext } from "./wallet.js";
+import type { WalletExecutionContext } from "./wallet.js";
 import {
   augmentPaytoUrisForWithdrawal,
   computeWithdrawalTransactionActions,
@@ -1487,9 +1487,6 @@ export async function getTransactions(
     x.txState.major === TransactionMajorState.Aborting ||
     x.txState.major === TransactionMajorState.Dialog;
 
-  const txPending = transactions.filter((x) => isPending(x));
-  const txNotPending = transactions.filter((x) => !isPending(x));
-
   let sortSign: number;
   if (transactionsRequest?.sort == "descending") {
     sortSign = -1;
@@ -1510,6 +1507,14 @@ export async function getTransactions(
     return sortSign * tsCmp;
   };
 
+  if (transactionsRequest?.sort === "stable-ascending") {
+    transactions.sort(txCmp);
+    return { transactions };
+  }
+
+  const txPending = transactions.filter((x) => isPending(x));
+  const txNotPending = transactions.filter((x) => !isPending(x));
+
   txPending.sort(txCmp);
   txNotPending.sort(txCmp);
 
diff --git a/packages/taler-wallet-core/src/withdraw.ts b/packages/taler-wallet-core/src/withdraw.ts
index 9132d2b09..625d5dca4 100644
--- a/packages/taler-wallet-core/src/withdraw.ts
+++ b/packages/taler-wallet-core/src/withdraw.ts
@@ -27,6 +27,7 @@ import {
   AcceptManualWithdrawalResult,
   AcceptWithdrawalResponse,
   AgeRestriction,
+  Amount,
   AmountJson,
   AmountLike,
   AmountString,
@@ -37,6 +38,7 @@ import {
   CoinStatus,
   CurrencySpecification,
   DenomKeyType,
+  DenomSelItem,
   DenomSelectionState,
   Duration,
   ExchangeBatchWithdrawRequest,
@@ -92,6 +94,7 @@ import {
   HttpResponse,
   readSuccessResponseJsonOrErrorCode,
   readSuccessResponseJsonOrThrow,
+  readTalerErrorResponse,
   throwUnexpectedRequestError,
 } from "@gnu-taler/taler-util/http";
 import {
@@ -665,15 +668,22 @@ async function processPlanchetGenerate(
     return;
   }
   let ci = 0;
+  let isSkipped = false;
   let maybeDenomPubHash: string | undefined;
   for (let di = 0; di < withdrawalGroup.denomsSel.selectedDenoms.length; di++) {
     const d = withdrawalGroup.denomsSel.selectedDenoms[di];
     if (coinIdx >= ci && coinIdx < ci + d.count) {
       maybeDenomPubHash = d.denomPubHash;
+      if (coinIdx >= ci + d.count - (d.skip ?? 0)) {
+        isSkipped = true;
+      }
       break;
     }
     ci += d.count;
   }
+  if (isSkipped) {
+    return;
+  }
   if (!maybeDenomPubHash) {
     throw Error("invariant violated");
   }
@@ -938,6 +948,9 @@ async function processPlanchetExchangeBatchRequest(
         logger.warn("processPlanchet: planchet already withdrawn");
         continue;
       }
+      if (planchet.planchetStatus === PlanchetStatus.AbortedReplaced) {
+        continue;
+      }
       const denom = await getDenomInfo(
         wex,
         tx,
@@ -968,10 +981,11 @@ async function processPlanchetExchangeBatchRequest(
     };
   }
 
-  async function storeCoinError(e: any, coinIdx: number): Promise<void> {
-    const errDetail = getErrorDetailFromException(e);
-    logger.trace("withdrawal request failed", e);
-    logger.trace(String(e));
+  async function storeCoinError(
+    errDetail: TalerErrorDetail,
+    coinIdx: number,
+  ): Promise<void> {
+    logger.trace(`withdrawal request failed: ${j2s(errDetail)}`);
     await wex.db.runReadWriteTx(["planchets"], async (tx) => {
       let planchet = await tx.planchets.indexes.byGroupAndIndex.get([
         withdrawalGroup.withdrawalGroupId,
@@ -1006,6 +1020,15 @@ async function processPlanchetExchangeBatchRequest(
         coinIdxs: [],
       };
     }
+    if (resp.status === HttpStatusCode.Gone) {
+      const e = await readTalerErrorResponse(resp);
+      // FIXME: Store in place of the planchet that is actually affected!
+      await storeCoinError(e, requestCoinIdxs[0]);
+      return {
+        batchResp: { ev_sigs: [] },
+        coinIdxs: [],
+      };
+    }
     const r = await readSuccessResponseJsonOrThrow(
       resp,
       codecForExchangeWithdrawBatchResponse(),
@@ -1015,7 +1038,10 @@ async function processPlanchetExchangeBatchRequest(
       batchResp: r,
     };
   } catch (e) {
-    await storeCoinError(e, requestCoinIdxs[0]);
+    const errDetail = getErrorDetailFromException(e);
+    // We don't know which coin is affected, so we store the error
+    // with the first coin of the batch.
+    await storeCoinError(errDetail, requestCoinIdxs[0]);
     return {
       batchResp: { ev_sigs: [] },
       coinIdxs: [],
@@ -1488,6 +1514,128 @@ async function processWithdrawalGroupPendingKyc(
   return TaskRunResult.backoff();
 }
 
+async function redenominateWithdrawal(
+  wex: WalletExecutionContext,
+  withdrawalGroupId: string,
+): Promise<void> {
+  logger.trace(`redenominating withdrawal group ${withdrawalGroupId}`);
+  await wex.db.runReadWriteTx(
+    ["withdrawalGroups", "planchets", "denominations"],
+    async (tx) => {
+      const wg = await tx.withdrawalGroups.get(withdrawalGroupId);
+      if (!wg) {
+        return;
+      }
+      const currency = Amounts.currencyOf(wg.denomsSel.totalWithdrawCost);
+      const exchangeBaseUrl = wg.exchangeBaseUrl;
+
+      const candidates = await getCandidateWithdrawalDenomsTx(
+        wex,
+        tx,
+        exchangeBaseUrl,
+        currency,
+      );
+
+      const oldSel = wg.denomsSel;
+
+      if (logger.shouldLogTrace()) {
+        logger.trace(`old denom sel: ${j2s(oldSel)}`);
+      }
+
+      let zero = Amount.zeroOfCurrency(currency);
+      let amountRemaining = zero;
+      let prevTotalCoinValue = zero;
+      let prevTotalWithdrawalCost = zero;
+      let prevDenoms: DenomSelItem[] = [];
+      let coinIndex = 0;
+      for (let i = 0; i < oldSel.selectedDenoms.length; i++) {
+        const sel = wg.denomsSel.selectedDenoms[i];
+        const denom = await tx.denominations.get([
+          exchangeBaseUrl,
+          sel.denomPubHash,
+        ]);
+        if (!denom) {
+          throw Error("denom in use but not not found");
+        }
+        // FIXME: Also check planchet if there was a different error or planchet already withdrawn
+        let denomOkay = isWithdrawableDenom(
+          denom,
+          wex.ws.config.testing.denomselAllowLate,
+        );
+        const numCoins = sel.count - (sel.skip ?? 0);
+        const denomValue = Amount.from(denom.value).mult(numCoins);
+        const denomFeeWithdraw = Amount.from(denom.fees.feeWithdraw).mult(
+          numCoins,
+        );
+        if (denomOkay) {
+          prevTotalCoinValue = prevTotalCoinValue.add(denomValue);
+          prevTotalWithdrawalCost = prevTotalWithdrawalCost.add(
+            denomValue,
+            denomFeeWithdraw,
+          );
+          prevDenoms.push({
+            count: sel.count,
+            denomPubHash: sel.denomPubHash,
+            skip: sel.skip,
+          });
+        } else {
+          amountRemaining = amountRemaining.add(denomValue, denomFeeWithdraw);
+          prevDenoms.push({
+            count: sel.count,
+            denomPubHash: sel.denomPubHash,
+            skip: (sel.skip ?? 0) + numCoins,
+          });
+
+          for (let j = 0; j < sel.count; j++) {
+            const ci = coinIndex + j;
+            const p = await tx.planchets.indexes.byGroupAndIndex.get([
+              withdrawalGroupId,
+              ci,
+            ]);
+            if (!p) {
+              // Maybe planchet wasn't yet generated.
+              // No problem!
+              logger.info(
+                `not aborting planchet #${coinIndex}, planchet not found`,
+              );
+              continue;
+            }
+            logger.info(`aborting planchet #${coinIndex}`);
+            p.planchetStatus = PlanchetStatus.AbortedReplaced;
+            await tx.planchets.put(p);
+          }
+        }
+
+        coinIndex += sel.count;
+      }
+
+      const newSel = selectWithdrawalDenominations(
+        amountRemaining.toJson(),
+        candidates,
+      );
+
+      if (logger.shouldLogTrace()) {
+        logger.trace(`new denom sel: ${j2s(newSel)}`);
+      }
+
+      const mergedSel: DenomSelectionState = {
+        selectedDenoms: [...prevDenoms, ...newSel.selectedDenoms],
+        totalCoinValue: zero
+          .add(prevTotalCoinValue, newSel.totalCoinValue)
+          .toString(),
+        totalWithdrawCost: zero
+          .add(prevTotalWithdrawalCost, newSel.totalWithdrawCost)
+          .toString(),
+      };
+      wg.denomsSel = mergedSel;
+      if (logger.shouldLogTrace()) {
+        logger.trace(`merged denom sel: ${j2s(mergedSel)}`);
+      }
+      await tx.withdrawalGroups.put(wg);
+    },
+  );
+}
+
 async function processWithdrawalGroupPendingReady(
   wex: WalletExecutionContext,
   withdrawalGroup: WithdrawalGroupRecord,
@@ -1498,6 +1646,8 @@ async function processWithdrawalGroupPendingReady(
     withdrawalGroupId,
   });
 
+  const exchangeBaseUrl = withdrawalGroup.exchangeBaseUrl;
+
   await fetchFreshExchange(wex, withdrawalGroup.exchangeBaseUrl);
 
   if (withdrawalGroup.denomsSel.selectedDenoms.length === 0) {
@@ -1576,9 +1726,41 @@ async function processWithdrawalGroupPendingReady(
     await Promise.all(work);
   }
 
-  let numFinished = 0;
+  let redenomRequired = false;
+
+  await wex.db.runReadOnlyTx(["planchets"], async (tx) => {
+    const planchets =
+      await tx.planchets.indexes.byGroup.getAll(withdrawalGroupId);
+    for (const p of planchets) {
+      if (p.planchetStatus !== PlanchetStatus.Pending) {
+        continue;
+      }
+      if (!p.lastError) {
+        continue;
+      }
+      switch (p.lastError.code) {
+        case TalerErrorCode.EXCHANGE_GENERIC_DENOMINATION_EXPIRED:
+        case TalerErrorCode.EXCHANGE_GENERIC_DENOMINATION_REVOKED:
+          redenomRequired = true;
+          return;
+      }
+    }
+  });
+
+  if (redenomRequired) {
+    logger.warn(`withdrawal ${withdrawalGroupId} requires redenomination`);
+    await fetchFreshExchange(wex, exchangeBaseUrl, {
+      forceUpdate: true,
+    });
+    await updateWithdrawalDenoms(wex, exchangeBaseUrl);
+    await redenominateWithdrawal(wex, withdrawalGroupId);
+    return TaskRunResult.backoff();
+  }
+
   const errorsPerCoin: Record<number, TalerErrorDetail> = {};
   let numPlanchetErrors = 0;
+  let numActive = 0;
+  let numDone = 0;
   const maxReportedErrors = 5;
 
   const res = await wex.db.runReadWriteTx(
@@ -1592,8 +1774,14 @@ async function processWithdrawalGroupPendingReady(
       await tx.planchets.indexes.byGroup
         .iter(withdrawalGroupId)
         .forEach((x) => {
-          if (x.planchetStatus === PlanchetStatus.WithdrawalDone) {
-            numFinished++;
+          switch (x.planchetStatus) {
+            case PlanchetStatus.KycRequired:
+            case PlanchetStatus.Pending:
+              numActive++;
+              break;
+            case PlanchetStatus.WithdrawalDone:
+              numDone++;
+              break;
           }
           if (x.lastError) {
             numPlanchetErrors++;
@@ -1603,8 +1791,8 @@ async function processWithdrawalGroupPendingReady(
           }
         });
       const oldTxState = computeWithdrawalTransactionStatus(wg);
-      logger.info(`now withdrawn ${numFinished} of ${numTotalCoins} coins`);
-      if (wg.timestampFinish === undefined && numFinished === numTotalCoins) {
+      logger.info(`now withdrawn ${numDone} of ${numTotalCoins} coins`);
+      if (wg.timestampFinish === undefined && numActive === 0) {
         wg.timestampFinish = timestampPreciseToDb(TalerPreciseTimestamp.now());
         wg.status = WithdrawalGroupStatus.Done;
         await makeCoinsVisible(wex, tx, transactionId);