implement backup scheduling, other tweaks

17 files changed, 329 insertions, 152 deletions

diff --git a/packages/taler-util/src/notifications.ts b/packages/taler-util/src/notifications.ts
index ade538d04..289dcb689 100644
--- a/packages/taler-util/src/notifications.ts
+++ b/packages/taler-util/src/notifications.ts
@@ -50,6 +50,7 @@ export enum NotificationType {
   RefundApplyOperationError = "refund-apply-error",
   RefundStatusOperationError = "refund-status-error",
   ProposalOperationError = "proposal-error",
+  BackupOperationError = "backup-error",
   TipOperationError = "tip-error",
   PayOperationError = "pay-error",
   PayOperationSuccess = "pay-operation-success",
@@ -159,6 +160,11 @@ export interface RefreshOperationErrorNotification {
   error: TalerErrorDetails;
 }
 
+export interface BackupOperationErrorNotification {
+  type: NotificationType.BackupOperationError;
+  error: TalerErrorDetails;
+}
+
 export interface RefundStatusOperationErrorNotification {
   type: NotificationType.RefundStatusOperationError;
   error: TalerErrorDetails;
@@ -234,6 +240,7 @@ export interface PayOperationSuccessNotification {
 }
 
 export type WalletNotification =
+  | BackupOperationErrorNotification
   | WithdrawOperationErrorNotification
   | ReserveOperationErrorNotification
   | ExchangeOperationErrorNotification
diff --git a/packages/taler-wallet-core/src/db.ts b/packages/taler-wallet-core/src/db.ts
index e640e7f20..2a2aba461 100644
--- a/packages/taler-wallet-core/src/db.ts
+++ b/packages/taler-wallet-core/src/db.ts
@@ -1552,11 +1552,26 @@ export interface RecoupGroupRecord {
   lastError: TalerErrorDetails | undefined;
 }
 
-export enum BackupProviderStatus {
-  PaymentRequired = "payment-required",
+export enum BackupProviderStateTag {
+  Provisional = "provisional",
   Ready = "ready",
+  Retrying = "retrying",
 }
 
+export type BackupProviderState =
+  | {
+      tag: BackupProviderStateTag.Provisional;
+    }
+  | {
+      tag: BackupProviderStateTag.Ready;
+      nextBackupTimestamp: Timestamp;
+    }
+  | {
+      tag: BackupProviderStateTag.Retrying;
+      retryInfo: RetryInfo;
+      lastError?: TalerErrorDetails;
+    };
+
 export interface BackupProviderTerms {
   supportedProtocolVersion: string;
   annualFee: AmountString;
@@ -1578,8 +1593,6 @@ export interface BackupProviderRecord {
    */
   terms?: BackupProviderTerms;
 
-  active: boolean;
-
   /**
    * Hash of the last encrypted backup that we already merged
    * or successfully uploaded ourselves.
@@ -1599,6 +1612,8 @@ export interface BackupProviderRecord {
    * Proposal that we're currently trying to pay for.
    *
    * (Also included in paymentProposalIds.)
+   *
+   * FIXME:  Make this part of a proper BackupProviderState?
    */
   currentPaymentProposalId?: string;
 
@@ -1610,20 +1625,7 @@ export interface BackupProviderRecord {
    */
   paymentProposalIds: string[];
 
-  /**
-   * Next scheduled backup.
-   */
-  nextBackupTimestamp?: Timestamp;
-
-  /**
-   * Retry info.
-   */
-  retryInfo: RetryInfo;
-
-  /**
-   * Last error that occurred, if any.
-   */
-  lastError: TalerErrorDetails | undefined;
+  state: BackupProviderState;
 
   /**
    * UIDs for the operation that added the backup provider.
@@ -1851,7 +1853,15 @@ export const WalletStoresV1 = {
     describeContents<BackupProviderRecord>("backupProviders", {
       keyPath: "baseUrl",
     }),
-    {},
+    {
+      byPaymentProposalId: describeIndex(
+        "byPaymentProposalId",
+        "paymentProposalIds",
+        {
+          multiEntry: true,
+        },
+      ),
+    },
   ),
   depositGroups: describeStore(
     describeContents<DepositGroupRecord>("depositGroups", {
diff --git a/packages/taler-wallet-core/src/operations/backup/import.ts b/packages/taler-wallet-core/src/operations/backup/import.ts
index b33e050b7..28bd5ec0a 100644
--- a/packages/taler-wallet-core/src/operations/backup/import.ts
+++ b/packages/taler-wallet-core/src/operations/backup/import.ts
@@ -263,7 +263,7 @@ export async function importBackup(
             updateClock: backupExchange.update_clock,
           },
           permanent: true,
-          retryInfo: initRetryInfo(false),
+          retryInfo: initRetryInfo(),
           lastUpdate: undefined,
           nextUpdate: getTimestampNow(),
           nextRefreshCheck: getTimestampNow(),
@@ -443,7 +443,7 @@ export async function importBackup(
               timestampReserveInfoPosted:
                 backupReserve.bank_info?.timestamp_reserve_info_posted,
               senderWire: backupReserve.sender_wire,
-              retryInfo: initRetryInfo(false),
+              retryInfo: initRetryInfo(),
               lastError: undefined,
               lastSuccessfulStatusQuery: { t_ms: "never" },
               initialWithdrawalGroupId:
@@ -483,7 +483,7 @@ export async function importBackup(
                   backupWg.raw_withdrawal_amount,
                 ),
                 reservePub,
-                retryInfo: initRetryInfo(false),
+                retryInfo: initRetryInfo(),
                 secretSeed: backupWg.secret_seed,
                 timestampStart: backupWg.timestamp_created,
                 timestampFinish: backupWg.timestamp_finish,
@@ -593,7 +593,7 @@ export async function importBackup(
               cryptoComp.proposalNoncePrivToPub[backupProposal.nonce_priv],
             proposalId: backupProposal.proposal_id,
             repurchaseProposalId: backupProposal.repurchase_proposal_id,
-            retryInfo: initRetryInfo(false),
+            retryInfo: initRetryInfo(),
             download,
             proposalStatus,
           });
@@ -728,7 +728,7 @@ export async function importBackup(
               cryptoComp.proposalNoncePrivToPub[backupPurchase.nonce_priv],
             lastPayError: undefined,
             autoRefundDeadline: { t_ms: "never" },
-            refundStatusRetryInfo: initRetryInfo(false),
+            refundStatusRetryInfo: initRetryInfo(),
             lastRefundStatusError: undefined,
             timestampAccept: backupPurchase.timestamp_accept,
             timestampFirstSuccessfulPay:
@@ -738,7 +738,7 @@ export async function importBackup(
             lastSessionId: undefined,
             abortStatus,
             // FIXME!
-            payRetryInfo: initRetryInfo(false),
+            payRetryInfo: initRetryInfo(),
             download,
             paymentSubmitPending: !backupPurchase.timestamp_first_successful_pay,
             refundQueryRequested: false,
@@ -835,7 +835,7 @@ export async function importBackup(
               Amounts.parseOrThrow(x.estimated_output_amount),
             ),
             refreshSessionPerCoin,
-            retryInfo: initRetryInfo(false),
+            retryInfo: initRetryInfo(),
           });
         }
       }
@@ -861,7 +861,7 @@ export async function importBackup(
             merchantBaseUrl: backupTip.exchange_base_url,
             merchantTipId: backupTip.merchant_tip_id,
             pickedUpTimestamp: backupTip.timestamp_finished,
-            retryInfo: initRetryInfo(false),
+            retryInfo: initRetryInfo(),
             secretSeed: backupTip.secret_seed,
             tipAmountEffective: denomsSel.totalCoinValue,
             tipAmountRaw: Amounts.parseOrThrow(backupTip.tip_amount_raw),
diff --git a/packages/taler-wallet-core/src/operations/backup/index.ts b/packages/taler-wallet-core/src/operations/backup/index.ts
index d367cf66a..68040695c 100644
--- a/packages/taler-wallet-core/src/operations/backup/index.ts
+++ b/packages/taler-wallet-core/src/operations/backup/index.ts
@@ -41,6 +41,7 @@ import {
   getTimestampNow,
   j2s,
   Logger,
+  NotificationType,
   PreparePayResultType,
   RecoveryLoadRequest,
   RecoveryMergeStrategy,
@@ -71,11 +72,15 @@ import {
 import { CryptoApi } from "../../crypto/workers/cryptoApi.js";
 import {
   BackupProviderRecord,
+  BackupProviderState,
+  BackupProviderStateTag,
   BackupProviderTerms,
   ConfigRecord,
   WalletBackupConfState,
+  WalletStoresV1,
   WALLET_BACKUP_STATE_KEY,
 } from "../../db.js";
+import { guardOperationException } from "../../errors.js";
 import {
   HttpResponseStatus,
   readSuccessResponseJsonOrThrow,
@@ -85,7 +90,8 @@ import {
   checkDbInvariant,
   checkLogicInvariant,
 } from "../../util/invariants.js";
-import { initRetryInfo } from "../../util/retries.js";
+import { GetReadWriteAccess } from "../../util/query.js";
+import { initRetryInfo, updateRetryInfoTimeout } from "../../util/retries.js";
 import {
   checkPaymentByProposalId,
   confirmPay,
@@ -247,6 +253,14 @@ interface BackupForProviderArgs {
   retryAfterPayment: boolean;
 }
 
+function getNextBackupTimestamp(): Timestamp {
+  // FIXME:  Randomize!
+  return timestampAddDuration(
+    getTimestampNow(),
+    durationFromSpec({ minutes: 5 }),
+  );
+}
+
 async function runBackupCycleForProvider(
   ws: InternalWalletState,
   args: BackupForProviderArgs,
@@ -304,8 +318,11 @@ async function runBackupCycleForProvider(
         if (!prov) {
           return;
         }
-        delete prov.lastError;
         prov.lastBackupCycleTimestamp = getTimestampNow();
+        prov.state = {
+          tag: BackupProviderStateTag.Ready,
+          nextBackupTimestamp: getNextBackupTimestamp(),
+        };
         await tx.backupProvider.put(prov);
       });
     return;
@@ -345,7 +362,9 @@ async function runBackupCycleForProvider(
         ids.add(proposalId);
         provRec.paymentProposalIds = Array.from(ids).sort();
         provRec.currentPaymentProposalId = proposalId;
+        // FIXME: allocate error code for this!
         await tx.backupProviders.put(provRec);
+        await incrementBackupRetryInTx(tx, args.provider.baseUrl, undefined);
       });
 
     if (doPay) {
@@ -376,7 +395,10 @@ async function runBackupCycleForProvider(
         }
         prov.lastBackupHash = encodeCrock(currentBackupHash);
         prov.lastBackupCycleTimestamp = getTimestampNow();
-        prov.lastError = undefined;
+        prov.state = {
+          tag: BackupProviderStateTag.Ready,
+          nextBackupTimestamp: getNextBackupTimestamp(),
+        };
         await tx.backupProviders.put(prov);
       });
     return;
@@ -397,11 +419,19 @@ async function runBackupCycleForProvider(
           return;
         }
         prov.lastBackupHash = encodeCrock(hash(backupEnc));
-        prov.lastBackupCycleTimestamp = getTimestampNow();
-        prov.lastError = undefined;
+        // FIXME:  Allocate error code for this situation?
+        prov.state = {
+          tag: BackupProviderStateTag.Retrying,
+          retryInfo: initRetryInfo(),
+        };
         await tx.backupProvider.put(prov);
       });
     logger.info("processed existing backup");
+    // Now upload our own, merged backup.
+    await runBackupCycleForProvider(ws, {
+      ...args,
+      retryAfterPayment: false,
+    });
     return;
   }
 
@@ -412,15 +442,82 @@ async function runBackupCycleForProvider(
   const err = await readTalerErrorResponse(resp);
   logger.error(`got error response from backup provider: ${j2s(err)}`);
   await ws.db
-    .mktx((x) => ({ backupProvider: x.backupProviders }))
+    .mktx((x) => ({ backupProviders: x.backupProviders }))
     .runReadWrite(async (tx) => {
-      const prov = await tx.backupProvider.get(provider.baseUrl);
-      if (!prov) {
-        return;
-      }
-      prov.lastError = err;
-      await tx.backupProvider.put(prov);
+      incrementBackupRetryInTx(tx, args.provider.baseUrl, err);
+    });
+}
+
+async function incrementBackupRetryInTx(
+  tx: GetReadWriteAccess<{
+    backupProviders: typeof WalletStoresV1.backupProviders;
+  }>,
+  backupProviderBaseUrl: string,
+  err: TalerErrorDetails | undefined,
+): Promise<void> {
+  const pr = await tx.backupProviders.get(backupProviderBaseUrl);
+  if (!pr) {
+    return;
+  }
+  if (pr.state.tag === BackupProviderStateTag.Retrying) {
+    pr.state.retryInfo.retryCounter++;
+    pr.state.lastError = err;
+    updateRetryInfoTimeout(pr.state.retryInfo);
+  } else if (pr.state.tag === BackupProviderStateTag.Ready) {
+    pr.state = {
+      tag: BackupProviderStateTag.Retrying,
+      retryInfo: initRetryInfo(),
+      lastError: err,
+    };
+  }
+  await tx.backupProviders.put(pr);
+}
+
+async function incrementBackupRetry(
+  ws: InternalWalletState,
+  backupProviderBaseUrl: string,
+  err: TalerErrorDetails | undefined,
+): Promise<void> {
+  await ws.db
+    .mktx((x) => ({ backupProviders: x.backupProviders }))
+    .runReadWrite(async (tx) =>
+      incrementBackupRetryInTx(tx, backupProviderBaseUrl, err),
+    );
+}
+
+export async function processBackupForProvider(
+  ws: InternalWalletState,
+  backupProviderBaseUrl: string,
+): Promise<void> {
+  const provider = await ws.db
+    .mktx((x) => ({ backupProviders: x.backupProviders }))
+    .runReadOnly(async (tx) => {
+      return await tx.backupProviders.get(backupProviderBaseUrl);
     });
+  if (!provider) {
+    throw Error("unknown backup provider");
+  }
+
+  const onOpErr = (err: TalerErrorDetails): Promise<void> =>
+    incrementBackupRetry(ws, backupProviderBaseUrl, err);
+
+  const run = async () => {
+    const backupJson = await exportBackup(ws);
+    const backupConfig = await provideBackupState(ws);
+    const encBackup = await encryptBackup(backupConfig, backupJson);
+    const currentBackupHash = hash(encBackup);
+
+    await runBackupCycleForProvider(ws, {
+      provider,
+      backupJson,
+      backupConfig,
+      encBackup,
+      currentBackupHash,
+      retryAfterPayment: true,
+    });
+  };
+
+  await guardOperationException(run, onOpErr);
 }
 
 /**
@@ -436,14 +533,9 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
     .runReadOnly(async (tx) => {
       return await tx.backupProviders.iter().toArray();
     });
-  logger.trace("got backup providers", providers);
   const backupJson = await exportBackup(ws);
-
-  logger.trace(`running backup cycle with backup JSON: ${j2s(backupJson)}`);
-
   const backupConfig = await provideBackupState(ws);
   const encBackup = await encryptBackup(backupConfig, backupJson);
-
   const currentBackupHash = hash(encBackup);
 
   for (const provider of providers) {
@@ -506,7 +598,10 @@ export async function addBackupProvider(
       if (oldProv) {
         logger.info("old backup provider found");
         if (req.activate) {
-          oldProv.active = true;
+          oldProv.state = {
+            tag: BackupProviderStateTag.Ready,
+            nextBackupTimestamp: getTimestampNow(),
+          };
           logger.info("setting existing backup provider to active");
           await tx.backupProviders.put(oldProv);
         }
@@ -522,8 +617,19 @@ export async function addBackupProvider(
   await ws.db
     .mktx((x) => ({ backupProviders: x.backupProviders }))
     .runReadWrite(async (tx) => {
+      let state: BackupProviderState;
+      if (req.activate) {
+        state = {
+          tag: BackupProviderStateTag.Ready,
+          nextBackupTimestamp: getTimestampNow(),
+        };
+      } else {
+        state = {
+          tag: BackupProviderStateTag.Provisional,
+        };
+      }
       await tx.backupProviders.put({
-        active: !!req.activate,
+        state,
         terms: {
           annualFee: terms.annual_fee,
           storageLimitInMegabytes: terms.storage_limit_in_megabytes,
@@ -531,8 +637,6 @@ export async function addBackupProvider(
         },
         paymentProposalIds: [],
         baseUrl: canonUrl,
-        lastError: undefined,
-        retryInfo: initRetryInfo(false),
         uids: [encodeCrock(getRandomBytes(32))],
       });
     });
@@ -697,11 +801,14 @@ export async function getBackupInfo(
   const providers: ProviderInfo[] = [];
   for (const x of providerRecords) {
     providers.push({
-      active: x.active,
+      active: x.state.tag !== BackupProviderStateTag.Provisional,
       syncProviderBaseUrl: x.baseUrl,
       lastSuccessfulBackupTimestamp: x.lastBackupCycleTimestamp,
       paymentProposalIds: x.paymentProposalIds,
-      lastError: x.lastError,
+      lastError:
+        x.state.tag === BackupProviderStateTag.Retrying
+          ? x.state.lastError
+          : undefined,
       paymentStatus: await getProviderPaymentInfo(ws, x),
       terms: x.terms,
     });
@@ -728,7 +835,7 @@ export async function getBackupRecovery(
     });
   return {
     providers: providers
-      .filter((x) => x.active)
+      .filter((x) => x.state.tag !== BackupProviderStateTag.Provisional)
       .map((x) => {
         return {
           url: x.baseUrl,
@@ -763,11 +870,12 @@ async function backupRecoveryTheirs(
         const existingProv = await tx.backupProviders.get(prov.url);
         if (!existingProv) {
           await tx.backupProviders.put({
-            active: true,
             baseUrl: prov.url,
             paymentProposalIds: [],
-            retryInfo: initRetryInfo(false),
-            lastError: undefined,
+            state: {
+              tag: BackupProviderStateTag.Ready,
+              nextBackupTimestamp: getTimestampNow(),
+            },
             uids: [encodeCrock(getRandomBytes(32))],
           });
         }
diff --git a/packages/taler-wallet-core/src/operations/deposits.ts b/packages/taler-wallet-core/src/operations/deposits.ts
index c788a9ea2..393919714 100644
--- a/packages/taler-wallet-core/src/operations/deposits.ts
+++ b/packages/taler-wallet-core/src/operations/deposits.ts
@@ -443,7 +443,7 @@ export async function createDepositGroup(
       payto_uri: req.depositPaytoUri,
       salt: wireSalt,
     },
-    retryInfo: initRetryInfo(true),
+    retryInfo: initRetryInfo(),
     lastError: undefined,
   };
 
diff --git a/packages/taler-wallet-core/src/operations/exchanges.ts b/packages/taler-wallet-core/src/operations/exchanges.ts
index a04769929..86a518671 100644
--- a/packages/taler-wallet-core/src/operations/exchanges.ts
+++ b/packages/taler-wallet-core/src/operations/exchanges.ts
@@ -297,7 +297,7 @@ async function provideExchangeRecord(
         r = {
           permanent: true,
           baseUrl: baseUrl,
-          retryInfo: initRetryInfo(false),
+          retryInfo: initRetryInfo(),
           detailsPointer: undefined,
           lastUpdate: undefined,
           nextUpdate: now,
@@ -498,7 +498,7 @@ async function updateExchangeFromUrlImpl(
       };
       // FIXME: only update if pointer got updated
       r.lastError = undefined;
-      r.retryInfo = initRetryInfo(false);
+      r.retryInfo = initRetryInfo();
       r.lastUpdate = getTimestampNow();
       (r.nextUpdate = keysInfo.expiry),
         // New denominations might be available.
diff --git a/packages/taler-wallet-core/src/operations/pay.ts b/packages/taler-wallet-core/src/operations/pay.ts
index 2cd3f7594..33d3bc83c 100644
--- a/packages/taler-wallet-core/src/operations/pay.ts
+++ b/packages/taler-wallet-core/src/operations/pay.ts
@@ -77,6 +77,7 @@ import {
   AbortStatus,
   AllowedAuditorInfo,
   AllowedExchangeInfo,
+  BackupProviderStateTag,
   CoinRecord,
   CoinStatus,
   DenominationRecord,
@@ -489,7 +490,7 @@ async function recordConfirmPay(
       if (p) {
         p.proposalStatus = ProposalStatus.ACCEPTED;
         delete p.lastError;
-        p.retryInfo = initRetryInfo(false);
+        p.retryInfo = initRetryInfo();
         await tx.proposals.put(p);
       }
       await tx.purchases.put(t);
@@ -942,7 +943,7 @@ async function storeFirstPaySuccess(
       purchase.paymentSubmitPending = false;
       purchase.lastPayError = undefined;
       purchase.lastSessionId = sessionId;
-      purchase.payRetryInfo = initRetryInfo(false);
+      purchase.payRetryInfo = initRetryInfo();
       purchase.merchantPaySig = paySig;
       if (isFirst) {
         const ar = purchase.download.contractData.autoRefund;
@@ -978,7 +979,7 @@ async function storePayReplaySuccess(
       }
       purchase.paymentSubmitPending = false;
       purchase.lastPayError = undefined;
-      purchase.payRetryInfo = initRetryInfo(false);
+      purchase.payRetryInfo = initRetryInfo();
       purchase.lastSessionId = sessionId;
       await tx.purchases.put(purchase);
     });
@@ -1100,6 +1101,26 @@ async function handleInsufficientFunds(
     });
 }
 
+async function unblockBackup(
+  ws: InternalWalletState,
+  proposalId: string,
+): Promise<void> {
+  await ws.db
+    .mktx((x) => ({ backupProviders: x.backupProviders }))
+    .runReadWrite(async (tx) => {
+      const bp = await tx.backupProviders.indexes.byPaymentProposalId
+        .iter(proposalId)
+        .forEachAsync(async (bp) => {
+          if (bp.state.tag === BackupProviderStateTag.Retrying) {
+            bp.state = {
+              tag: BackupProviderStateTag.Ready,
+              nextBackupTimestamp: getTimestampNow(),
+            };
+          }
+        });
+    });
+}
+
 /**
  * Submit a payment to the merchant.
  *
@@ -1228,6 +1249,7 @@ async function submitPay(
     }
 
     await storeFirstPaySuccess(ws, proposalId, sessionId, merchantResp.sig);
+    await unblockBackup(ws, proposalId);
   } else {
     const payAgainUrl = new URL(
       `orders/${purchase.download.contractData.orderId}/paid`,
@@ -1266,6 +1288,7 @@ async function submitPay(
       );
     }
     await storePayReplaySuccess(ws, proposalId, sessionId);
+    await unblockBackup(ws, proposalId);
   }
 
   ws.notify({
diff --git a/packages/taler-wallet-core/src/operations/pending.ts b/packages/taler-wallet-core/src/operations/pending.ts
index fff64739c..3a6af186e 100644
--- a/packages/taler-wallet-core/src/operations/pending.ts
+++ b/packages/taler-wallet-core/src/operations/pending.ts
@@ -15,6 +15,10 @@
  */
 
 /**
+ * Derive pending tasks from the wallet database.
+ */
+
+/**
  * Imports.
  */
 import {
@@ -22,13 +26,18 @@ import {
   ReserveRecordStatus,
   AbortStatus,
   WalletStoresV1,
+  BackupProviderStateTag,
 } from "../db.js";
 import {
   PendingOperationsResponse,
-  PendingOperationType,
+  PendingTaskType,
   ReserveType,
 } from "../pending-types.js";
-import { getTimestampNow, Timestamp } from "@gnu-taler/taler-util";
+import {
+  getTimestampNow,
+  isTimestampExpired,
+  Timestamp,
+} from "@gnu-taler/taler-util";
 import { InternalWalletState } from "../common.js";
 import { getBalancesInsideTransaction } from "./balance.js";
 import { GetReadOnlyAccess } from "../util/query.js";
@@ -43,7 +52,7 @@ async function gatherExchangePending(
 ): Promise<void> {
   await tx.exchanges.iter().forEachAsync(async (e) => {
     resp.pendingOperations.push({
-      type: PendingOperationType.ExchangeUpdate,
+      type: PendingTaskType.ExchangeUpdate,
       givesLifeness: false,
       timestampDue: e.nextUpdate,
       exchangeBaseUrl: e.baseUrl,
@@ -51,7 +60,7 @@ async function gatherExchangePending(
     });
 
     resp.pendingOperations.push({
-      type: PendingOperationType.ExchangeCheckRefresh,
+      type: PendingTaskType.ExchangeCheckRefresh,
       timestampDue: e.nextRefreshCheck,
       givesLifeness: false,
       exchangeBaseUrl: e.baseUrl,
@@ -76,7 +85,7 @@ async function gatherReservePending(
       case ReserveRecordStatus.QUERYING_STATUS:
       case ReserveRecordStatus.REGISTERING_BANK:
         resp.pendingOperations.push({
-          type: PendingOperationType.Reserve,
+          type: PendingTaskType.Reserve,
           givesLifeness: true,
           timestampDue: reserve.retryInfo.nextRetry,
           stage: reserve.reserveStatus,
@@ -103,7 +112,7 @@ async function gatherRefreshPending(
       return;
     }
     resp.pendingOperations.push({
-      type: PendingOperationType.Refresh,
+      type: PendingTaskType.Refresh,
       givesLifeness: true,
       timestampDue: r.retryInfo.nextRetry,
       refreshGroupId: r.refreshGroupId,
@@ -136,7 +145,7 @@ async function gatherWithdrawalPending(
         }
       });
     resp.pendingOperations.push({
-      type: PendingOperationType.Withdraw,
+      type: PendingTaskType.Withdraw,
       givesLifeness: true,
       timestampDue: wsr.retryInfo.nextRetry,
       withdrawalGroupId: wsr.withdrawalGroupId,
@@ -157,7 +166,7 @@ async function gatherProposalPending(
     } else if (proposal.proposalStatus == ProposalStatus.DOWNLOADING) {
       const timestampDue = proposal.retryInfo?.nextRetry ?? getTimestampNow();
       resp.pendingOperations.push({
-        type: PendingOperationType.ProposalDownload,
+        type: PendingTaskType.ProposalDownload,
         givesLifeness: true,
         timestampDue,
         merchantBaseUrl: proposal.merchantBaseUrl,
@@ -182,7 +191,7 @@ async function gatherTipPending(
     }
     if (tip.acceptedTimestamp) {
       resp.pendingOperations.push({
-        type: PendingOperationType.TipPickup,
+        type: PendingTaskType.TipPickup,
         givesLifeness: true,
         timestampDue: tip.retryInfo.nextRetry,
         merchantBaseUrl: tip.merchantBaseUrl,
@@ -202,7 +211,7 @@ async function gatherPurchasePending(
     if (pr.paymentSubmitPending && pr.abortStatus === AbortStatus.None) {
       const timestampDue = pr.payRetryInfo?.nextRetry ?? getTimestampNow();
       resp.pendingOperations.push({
-        type: PendingOperationType.Pay,
+        type: PendingTaskType.Pay,
         givesLifeness: true,
         timestampDue,
         isReplay: false,
@@ -213,7 +222,7 @@ async function gatherPurchasePending(
     }
     if (pr.refundQueryRequested) {
       resp.pendingOperations.push({
-        type: PendingOperationType.RefundQuery,
+        type: PendingTaskType.RefundQuery,
         givesLifeness: true,
         timestampDue: pr.refundStatusRetryInfo.nextRetry,
         proposalId: pr.proposalId,
@@ -234,7 +243,7 @@ async function gatherRecoupPending(
       return;
     }
     resp.pendingOperations.push({
-      type: PendingOperationType.Recoup,
+      type: PendingTaskType.Recoup,
       givesLifeness: true,
       timestampDue: rg.retryInfo.nextRetry,
       recoupGroupId: rg.recoupGroupId,
@@ -244,23 +253,32 @@ async function gatherRecoupPending(
   });
 }
 
-async function gatherDepositPending(
-  tx: GetReadOnlyAccess<{ depositGroups: typeof WalletStoresV1.depositGroups }>,
+async function gatherBackupPending(
+  tx: GetReadOnlyAccess<{
+    backupProviders: typeof WalletStoresV1.backupProviders;
+  }>,
   now: Timestamp,
   resp: PendingOperationsResponse,
 ): Promise<void> {
-  await tx.depositGroups.iter().forEach((dg) => {
-    if (dg.timestampFinished) {
-      return;
+  await tx.backupProviders.iter().forEach((bp) => {
+    if (bp.state.tag === BackupProviderStateTag.Ready) {
+      resp.pendingOperations.push({
+        type: PendingTaskType.Backup,
+        givesLifeness: false,
+        timestampDue: bp.state.nextBackupTimestamp,
+        backupProviderBaseUrl: bp.baseUrl,
+        lastError: undefined,
+      });
+    } else if (bp.state.tag === BackupProviderStateTag.Retrying) {
+      resp.pendingOperations.push({
+        type: PendingTaskType.Backup,
+        givesLifeness: false,
+        timestampDue: bp.state.retryInfo.nextRetry,
+        backupProviderBaseUrl: bp.baseUrl,
+        retryInfo: bp.state.retryInfo,
+        lastError: bp.state.lastError,
+      });
     }
-    resp.pendingOperations.push({
-      type: PendingOperationType.Deposit,
-      givesLifeness: true,
-      timestampDue: dg.retryInfo.nextRetry,
-      depositGroupId: dg.depositGroupId,
-      retryInfo: dg.retryInfo,
-      lastError: dg.lastError,
-    });
   });
 }
 
@@ -270,6 +288,7 @@ export async function getPendingOperations(
   const now = getTimestampNow();
   return await ws.db
     .mktx((x) => ({
+      backupProviders: x.backupProviders,
       exchanges: x.exchanges,
       exchangeDetails: x.exchangeDetails,
       reserves: x.reserves,
@@ -297,7 +316,7 @@ export async function getPendingOperations(
       await gatherTipPending(tx, now, resp);
       await gatherPurchasePending(tx, now, resp);
       await gatherRecoupPending(tx, now, resp);
-      await gatherDepositPending(tx, now, resp);
+      await gatherBackupPending(tx, now, resp);
       return resp;
     });
 }
diff --git a/packages/taler-wallet-core/src/operations/recoup.ts b/packages/taler-wallet-core/src/operations/recoup.ts
index 4510bda10..634469923 100644
--- a/packages/taler-wallet-core/src/operations/recoup.ts
+++ b/packages/taler-wallet-core/src/operations/recoup.ts
@@ -109,7 +109,7 @@ async function putGroupAsFinished(
   if (allFinished) {
     logger.trace("all recoups of recoup group are finished");
     recoupGroup.timestampFinished = getTimestampNow();
-    recoupGroup.retryInfo = initRetryInfo(false);
+    recoupGroup.retryInfo = initRetryInfo();
     recoupGroup.lastError = undefined;
     if (recoupGroup.scheduleRefreshCoins.length > 0) {
       const refreshGroupId = await createRefreshGroup(
diff --git a/packages/taler-wallet-core/src/operations/refresh.ts b/packages/taler-wallet-core/src/operations/refresh.ts
index cf8b4ddde..2549b1404 100644
--- a/packages/taler-wallet-core/src/operations/refresh.ts
+++ b/packages/taler-wallet-core/src/operations/refresh.ts
@@ -203,7 +203,7 @@ async function refreshCreateSession(
         }
         if (allDone) {
           rg.timestampFinished = getTimestampNow();
-          rg.retryInfo = initRetryInfo(false);
+          rg.retryInfo = initRetryInfo();
         }
         await tx.refreshGroups.put(rg);
       });
@@ -590,7 +590,7 @@ async function refreshReveal(
       }
       if (allDone) {
         rg.timestampFinished = getTimestampNow();
-        rg.retryInfo = initRetryInfo(false);
+        rg.retryInfo = initRetryInfo();
       }
       for (const coin of coins) {
         await tx.coins.put(coin);
diff --git a/packages/taler-wallet-core/src/operations/refund.ts b/packages/taler-wallet-core/src/operations/refund.ts
index 0bff29863..a5846f259 100644
--- a/packages/taler-wallet-core/src/operations/refund.ts
+++ b/packages/taler-wallet-core/src/operations/refund.ts
@@ -405,7 +405,7 @@ async function acceptRefunds(
       if (queryDone) {
         p.timestampLastRefundStatus = now;
         p.lastRefundStatusError = undefined;
-        p.refundStatusRetryInfo = initRetryInfo(false);
+        p.refundStatusRetryInfo = initRetryInfo();
         p.refundQueryRequested = false;
         if (p.abortStatus === AbortStatus.AbortRefund) {
           p.abortStatus = AbortStatus.AbortFinished;
@@ -768,7 +768,7 @@ export async function abortFailedPayWithRefund(
       purchase.paymentSubmitPending = false;
       purchase.abortStatus = AbortStatus.AbortRefund;
       purchase.lastPayError = undefined;
-      purchase.payRetryInfo = initRetryInfo(false);
+      purchase.payRetryInfo = initRetryInfo();
       await tx.purchases.put(purchase);
     });
   processPurchaseQueryRefund(ws, proposalId, true).catch((e) => {
diff --git a/packages/taler-wallet-core/src/operations/reserves.ts b/packages/taler-wallet-core/src/operations/reserves.ts
index 162b5b405..a3536eed6 100644
--- a/packages/taler-wallet-core/src/operations/reserves.ts
+++ b/packages/taler-wallet-core/src/operations/reserves.ts
@@ -651,7 +651,7 @@ async function updateReserve(
       if (denomSelInfo.selectedDenoms.length === 0) {
         newReserve.reserveStatus = ReserveRecordStatus.DORMANT;
         newReserve.lastError = undefined;
-        newReserve.retryInfo = initRetryInfo(false);
+        newReserve.retryInfo = initRetryInfo();
         await tx.reserves.put(newReserve);
         return;
       }
@@ -679,7 +679,7 @@ async function updateReserve(
       };
 
       newReserve.lastError = undefined;
-      newReserve.retryInfo = initRetryInfo(false);
+      newReserve.retryInfo = initRetryInfo();
       newReserve.reserveStatus = ReserveRecordStatus.DORMANT;
 
       await tx.reserves.put(newReserve);
diff --git a/packages/taler-wallet-core/src/operations/tip.ts b/packages/taler-wallet-core/src/operations/tip.ts
index 892a3b588..29eeb8d59 100644
--- a/packages/taler-wallet-core/src/operations/tip.ts
+++ b/packages/taler-wallet-core/src/operations/tip.ts
@@ -388,7 +388,7 @@ async function processTipImpl(
       }
       tr.pickedUpTimestamp = getTimestampNow();
       tr.lastError = undefined;
-      tr.retryInfo = initRetryInfo(false);
+      tr.retryInfo = initRetryInfo();
       await tx.tips.put(tr);
       for (const cr of newCoinRecords) {
         await tx.coins.put(cr);
diff --git a/packages/taler-wallet-core/src/operations/withdraw.ts b/packages/taler-wallet-core/src/operations/withdraw.ts
index e966f6a14..55f39b6bf 100644
--- a/packages/taler-wallet-core/src/operations/withdraw.ts
+++ b/packages/taler-wallet-core/src/operations/withdraw.ts
@@ -875,7 +875,7 @@ async function processWithdrawGroupImpl(
         finishedForFirstTime = true;
         wg.timestampFinish = getTimestampNow();
         wg.lastError = undefined;
-        wg.retryInfo = initRetryInfo(false);
+        wg.retryInfo = initRetryInfo();
       }
 
       await tx.withdrawalGroups.put(wg);
diff --git a/packages/taler-wallet-core/src/pending-types.ts b/packages/taler-wallet-core/src/pending-types.ts
index 0e26c262b..505220e72 100644
--- a/packages/taler-wallet-core/src/pending-types.ts
+++ b/packages/taler-wallet-core/src/pending-types.ts
@@ -15,9 +15,9 @@
  */
 
 /**
- * Type and schema definitions for pending operations in the wallet.
+ * Type and schema definitions for pending tasks in the wallet.
  *
- * These are only used internally, and are not part of the public
+ * These are only used internally, and are not part of the stable public
  * interface to the wallet.
  */
 
@@ -32,7 +32,7 @@ import {
 import { ReserveRecordStatus } from "./db.js";
 import { RetryInfo } from "./util/retries.js";
 
-export enum PendingOperationType {
+export enum PendingTaskType {
   ExchangeUpdate = "exchange-update",
   ExchangeCheckRefresh = "exchange-check-refresh",
   Pay = "pay",
@@ -45,31 +45,39 @@ export enum PendingOperationType {
   TipPickup = "tip-pickup",
   Withdraw = "withdraw",
   Deposit = "deposit",
+  Backup = "backup",
 }
 
 /**
  * Information about a pending operation.
  */
-export type PendingOperationInfo = PendingOperationInfoCommon &
+export type PendingTaskInfo = PendingTaskInfoCommon &
   (
-    | PendingExchangeUpdateOperation
-    | PendingExchangeCheckRefreshOperation
-    | PendingPayOperation
-    | PendingProposalDownloadOperation
-    | PendingRefreshOperation
-    | PendingRefundQueryOperation
-    | PendingReserveOperation
-    | PendingTipPickupOperation
-    | PendingWithdrawOperation
-    | PendingRecoupOperation
-    | PendingDepositOperation
+    | PendingExchangeUpdateTask
+    | PendingExchangeCheckRefreshTask
+    | PendingPayTask
+    | PendingProposalDownloadTask
+    | PendingRefreshTask
+    | PendingRefundQueryTask
+    | PendingReserveTask
+    | PendingTipPickupTask
+    | PendingWithdrawTask
+    | PendingRecoupTask
+    | PendingDepositTask
+    | PendingBackupTask
   );
 
+export interface PendingBackupTask {
+  type: PendingTaskType.Backup;
+  backupProviderBaseUrl: string;
+  lastError: TalerErrorDetails | undefined;
+}
+
 /**
  * The wallet is currently updating information about an exchange.
  */
-export interface PendingExchangeUpdateOperation {
-  type: PendingOperationType.ExchangeUpdate;
+export interface PendingExchangeUpdateTask {
+  type: PendingTaskType.ExchangeUpdate;
   exchangeBaseUrl: string;
   lastError: TalerErrorDetails | undefined;
 }
@@ -78,8 +86,8 @@ export interface PendingExchangeUpdateOperation {
  * The wallet should check whether coins from this exchange
  * need to be auto-refreshed.
  */
-export interface PendingExchangeCheckRefreshOperation {
-  type: PendingOperationType.ExchangeCheckRefresh;
+export interface PendingExchangeCheckRefreshTask {
+  type: PendingTaskType.ExchangeCheckRefresh;
   exchangeBaseUrl: string;
 }
 
@@ -100,8 +108,8 @@ export enum ReserveType {
  * Does *not* include the withdrawal operation that might result
  * from this.
  */
-export interface PendingReserveOperation {
-  type: PendingOperationType.Reserve;
+export interface PendingReserveTask {
+  type: PendingTaskType.Reserve;
   retryInfo: RetryInfo | undefined;
   stage: ReserveRecordStatus;
   timestampCreated: Timestamp;
@@ -113,8 +121,8 @@ export interface PendingReserveOperation {
 /**
  * Status of an ongoing withdrawal operation.
  */
-export interface PendingRefreshOperation {
-  type: PendingOperationType.Refresh;
+export interface PendingRefreshTask {
+  type: PendingTaskType.Refresh;
   lastError?: TalerErrorDetails;
   refreshGroupId: string;
   finishedPerCoin: boolean[];
@@ -124,8 +132,8 @@ export interface PendingRefreshOperation {
 /**
  * Status of downloading signed contract terms from a merchant.
  */
-export interface PendingProposalDownloadOperation {
-  type: PendingOperationType.ProposalDownload;
+export interface PendingProposalDownloadTask {
+  type: PendingTaskType.ProposalDownload;
   merchantBaseUrl: string;
   proposalTimestamp: Timestamp;
   proposalId: string;
@@ -139,7 +147,7 @@ export interface PendingProposalDownloadOperation {
  * proposed contract terms.
  */
 export interface PendingProposalChoiceOperation {
-  type: PendingOperationType.ProposalChoice;
+  type: PendingTaskType.ProposalChoice;
   merchantBaseUrl: string;
   proposalTimestamp: Timestamp;
   proposalId: string;
@@ -148,8 +156,8 @@ export interface PendingProposalChoiceOperation {
 /**
  * The wallet is picking up a tip that the user has accepted.
  */
-export interface PendingTipPickupOperation {
-  type: PendingOperationType.TipPickup;
+export interface PendingTipPickupTask {
+  type: PendingTaskType.TipPickup;
   tipId: string;
   merchantBaseUrl: string;
   merchantTipId: string;
@@ -159,8 +167,8 @@ export interface PendingTipPickupOperation {
  * The wallet is signing coins and then sending them to
  * the merchant.
  */
-export interface PendingPayOperation {
-  type: PendingOperationType.Pay;
+export interface PendingPayTask {
+  type: PendingTaskType.Pay;
   proposalId: string;
   isReplay: boolean;
   retryInfo?: RetryInfo;
@@ -171,15 +179,15 @@ export interface PendingPayOperation {
  * The wallet is querying the merchant about whether any refund
  * permissions are available for a purchase.
  */
-export interface PendingRefundQueryOperation {
-  type: PendingOperationType.RefundQuery;
+export interface PendingRefundQueryTask {
+  type: PendingTaskType.RefundQuery;
   proposalId: string;
   retryInfo: RetryInfo;
   lastError: TalerErrorDetails | undefined;
 }
 
-export interface PendingRecoupOperation {
-  type: PendingOperationType.Recoup;
+export interface PendingRecoupTask {
+  type: PendingTaskType.Recoup;
   recoupGroupId: string;
   retryInfo: RetryInfo;
   lastError: TalerErrorDetails | undefined;
@@ -188,8 +196,8 @@ export interface PendingRecoupOperation {
 /**
  * Status of an ongoing withdrawal operation.
  */
-export interface PendingWithdrawOperation {
-  type: PendingOperationType.Withdraw;
+export interface PendingWithdrawTask {
+  type: PendingTaskType.Withdraw;
   lastError: TalerErrorDetails | undefined;
   retryInfo: RetryInfo;
   withdrawalGroupId: string;
@@ -198,8 +206,8 @@ export interface PendingWithdrawOperation {
 /**
  * Status of an ongoing deposit operation.
  */
-export interface PendingDepositOperation {
-  type: PendingOperationType.Deposit;
+export interface PendingDepositTask {
+  type: PendingTaskType.Deposit;
   lastError: TalerErrorDetails | undefined;
   retryInfo: RetryInfo;
   depositGroupId: string;
@@ -208,11 +216,11 @@ export interface PendingDepositOperation {
 /**
  * Fields that are present in every pending operation.
  */
-export interface PendingOperationInfoCommon {
+export interface PendingTaskInfoCommon {
   /**
    * Type of the pending operation.
    */
-  type: PendingOperationType;
+  type: PendingTaskType;
 
   /**
    * Set to true if the operation indicates that something is really in progress,
@@ -239,7 +247,7 @@ export interface PendingOperationsResponse {
   /**
    * List of pending operations.
    */
-  pendingOperations: PendingOperationInfo[];
+  pendingOperations: PendingTaskInfo[];
 
   /**
    * Current wallet balance, including pending balances.
diff --git a/packages/taler-wallet-core/src/util/retries.ts b/packages/taler-wallet-core/src/util/retries.ts
index b86846244..cac7b1b52 100644
--- a/packages/taler-wallet-core/src/util/retries.ts
+++ b/packages/taler-wallet-core/src/util/retries.ts
@@ -72,13 +72,11 @@ export function getRetryDuration(
 }
 
 export function initRetryInfo(
-  active = true,
   p: RetryPolicy = defaultRetryPolicy,
 ): RetryInfo {
   const now = getTimestampNow();
   const info = {
     firstTry: now,
-    active: true,
     nextRetry: now,
     retryCounter: 0,
   };
diff --git a/packages/taler-wallet-core/src/wallet.ts b/packages/taler-wallet-core/src/wallet.ts
index de0675cd6..ca9afc073 100644
--- a/packages/taler-wallet-core/src/wallet.ts
+++ b/packages/taler-wallet-core/src/wallet.ts
@@ -44,6 +44,7 @@ import {
   getBackupInfo,
   getBackupRecovery,
   loadBackupRecovery,
+  processBackupForProvider,
   runBackupCycle,
 } from "./operations/backup/index.js";
 import { exportBackup } from "./operations/backup/export.js";
@@ -118,9 +119,9 @@ import {
 } from "./db.js";
 import { NotificationType } from "@gnu-taler/taler-util";
 import {
-  PendingOperationInfo,
+  PendingTaskInfo,
   PendingOperationsResponse,
-  PendingOperationType,
+  PendingTaskType,
 } from "./pending-types.js";
 import { CoinDumpJson } from "@gnu-taler/taler-util";
 import { codecForTransactionsRequest } from "@gnu-taler/taler-util";
@@ -206,44 +207,47 @@ async function getWithdrawalDetailsForAmount(
  */
 async function processOnePendingOperation(
   ws: InternalWalletState,
-  pending: PendingOperationInfo,
+  pending: PendingTaskInfo,
   forceNow = false,
 ): Promise<void> {
   logger.trace(`running pending ${JSON.stringify(pending, undefined, 2)}`);
   switch (pending.type) {
-    case PendingOperationType.ExchangeUpdate:
+    case PendingTaskType.ExchangeUpdate:
       await updateExchangeFromUrl(ws, pending.exchangeBaseUrl, forceNow);
       break;
-    case PendingOperationType.Refresh:
+    case PendingTaskType.Refresh:
       await processRefreshGroup(ws, pending.refreshGroupId, forceNow);
       break;
-    case PendingOperationType.Reserve:
+    case PendingTaskType.Reserve:
       await processReserve(ws, pending.reservePub, forceNow);
       break;
-    case PendingOperationType.Withdraw:
+    case PendingTaskType.Withdraw:
       await processWithdrawGroup(ws, pending.withdrawalGroupId, forceNow);
       break;
-    case PendingOperationType.ProposalDownload:
+    case PendingTaskType.ProposalDownload:
       await processDownloadProposal(ws, pending.proposalId, forceNow);
       break;
-    case PendingOperationType.TipPickup:
+    case PendingTaskType.TipPickup:
       await processTip(ws, pending.tipId, forceNow);
       break;
-    case PendingOperationType.Pay:
+    case PendingTaskType.Pay:
       await processPurchasePay(ws, pending.proposalId, forceNow);
       break;
-    case PendingOperationType.RefundQuery:
+    case PendingTaskType.RefundQuery:
       await processPurchaseQueryRefund(ws, pending.proposalId, forceNow);
       break;
-    case PendingOperationType.Recoup:
+    case PendingTaskType.Recoup:
       await processRecoupGroup(ws, pending.recoupGroupId, forceNow);
       break;
-    case PendingOperationType.ExchangeCheckRefresh:
+    case PendingTaskType.ExchangeCheckRefresh:
       await autoRefresh(ws, pending.exchangeBaseUrl);
       break;
-    case PendingOperationType.Deposit:
+    case PendingTaskType.Deposit:
       await processDepositGroup(ws, pending.depositGroupId);
       break;
+    case PendingTaskType.Backup:
+      await processBackupForProvider(ws, pending.backupProviderBaseUrl);
+      break;
     default:
       assertUnreachable(pending);
   }