finish first complete end-to-end backup/sync test

3 files changed, 322 insertions, 199 deletions

diff --git a/packages/taler-wallet-core/src/operations/backup/import.ts b/packages/taler-wallet-core/src/operations/backup/import.ts
index fa0819745..416b068e4 100644
--- a/packages/taler-wallet-core/src/operations/backup/import.ts
+++ b/packages/taler-wallet-core/src/operations/backup/import.ts
@@ -15,68 +15,47 @@
  */
 
 import {
-  Stores,
-  Amounts,
-  CoinSourceType,
-  CoinStatus,
-  RefundState,
   AbortStatus,
-  ProposalStatus,
-  getTimestampNow,
-  encodeCrock,
-  stringToBytes,
-  getRandomBytes,
   AmountJson,
+  Amounts,
   codecForContractTerms,
   CoinSource,
+  CoinSourceType,
+  CoinStatus,
   DenominationStatus,
   DenomSelectionState,
   ExchangeUpdateStatus,
   ExchangeWireInfo,
+  getTimestampNow,
   PayCoinSelection,
   ProposalDownload,
+  ProposalStatus,
   RefreshReason,
   RefreshSessionRecord,
+  RefundState,
   ReserveBankInfo,
   ReserveRecordStatus,
+  Stores,
   TransactionHandle,
   WalletContractData,
   WalletRefundItem,
 } from "../..";
-import { hash } from "../../crypto/primitives/nacl-fast";
 import {
-  WalletBackupContentV1,
-  BackupExchange,
-  BackupCoin,
-  BackupDenomination,
-  BackupReserve,
-  BackupPurchase,
-  BackupProposal,
-  BackupRefreshGroup,
-  BackupBackupProvider,
-  BackupTip,
-  BackupRecoupGroup,
-  BackupWithdrawalGroup,
-  BackupBackupProviderTerms,
-  BackupCoinSource,
   BackupCoinSourceType,
-  BackupExchangeWireFee,
-  BackupRefundItem,
-  BackupRefundState,
-  BackupProposalStatus,
-  BackupRefreshOldCoin,
-  BackupRefreshSession,
   BackupDenomSel,
+  BackupProposalStatus,
+  BackupPurchase,
   BackupRefreshReason,
+  BackupRefundState,
+  WalletBackupContentV1,
 } from "../../types/backupTypes";
-import { canonicalizeBaseUrl, canonicalJson, j2s } from "../../util/helpers";
+import { j2s } from "../../util/helpers";
 import { checkDbInvariant, checkLogicInvariant } from "../../util/invariants";
 import { Logger } from "../../util/logging";
 import { initRetryInfo } from "../../util/retries";
 import { InternalWalletState } from "../state";
 import { provideBackupState } from "./state";
 
-
 const logger = new Logger("operations/backup/import.ts");
 
 function checkBackupInvariant(b: boolean, m?: string): asserts b {
@@ -230,6 +209,9 @@ export async function importBackup(
   cryptoComp: BackupCryptoPrecomputedData,
 ): Promise<void> {
   await provideBackupState(ws);
+
+  logger.info(`importing backup ${j2s(backupBlobArg)}`);
+
   return ws.db.runWithWriteTransaction(
     [
       Stores.config,
diff --git a/packages/taler-wallet-core/src/operations/backup/index.ts b/packages/taler-wallet-core/src/operations/backup/index.ts
index fd0274219..edc5acc15 100644
--- a/packages/taler-wallet-core/src/operations/backup/index.ts
+++ b/packages/taler-wallet-core/src/operations/backup/index.ts
@@ -27,7 +27,11 @@
 import { InternalWalletState } from "../state";
 import { WalletBackupContentV1 } from "../../types/backupTypes";
 import { TransactionHandle } from "../../util/query";
-import { ConfigRecord, Stores } from "../../types/dbTypes";
+import {
+  BackupProviderRecord,
+  ConfigRecord,
+  Stores,
+} from "../../types/dbTypes";
 import { checkDbInvariant, checkLogicInvariant } from "../../util/invariants";
 import { codecForAmountString } from "../../util/amounts";
 import {
@@ -41,7 +45,13 @@ import {
   stringToBytes,
 } from "../../crypto/talerCrypto";
 import { canonicalizeBaseUrl, canonicalJson, j2s } from "../../util/helpers";
-import { getTimestampNow, Timestamp } from "../../util/time";
+import {
+  durationAdd,
+  durationFromSpec,
+  getTimestampNow,
+  Timestamp,
+  timestampAddDuration,
+} from "../../util/time";
 import { URL } from "../../util/url";
 import { AmountString } from "../../types/talerTypes";
 import {
@@ -70,7 +80,7 @@ import {
 } from "../../types/walletTypes";
 import { CryptoApi } from "../../crypto/workers/cryptoApi";
 import { secretbox, secretbox_open } from "../../crypto/primitives/nacl-fast";
-import { confirmPay, preparePayForUri } from "../pay";
+import { checkPaymentByProposalId, confirmPay, preparePayForUri } from "../pay";
 import { exportBackup } from "./export";
 import { BackupCryptoPrecomputedData, importBackup } from "./import";
 import {
@@ -79,6 +89,7 @@ import {
   getWalletBackupState,
   WalletBackupConfState,
 } from "./state";
+import { PaymentStatus } from "../../types/transactionsTypes";
 
 const logger = new Logger("operations/backup.ts");
 
@@ -216,93 +227,103 @@ function deriveBlobSecret(bc: WalletBackupConfState): Uint8Array {
   );
 }
 
-/**
- * Do one backup cycle that consists of:
- * 1. Exporting a backup and try to upload it.
- *    Stop if this step succeeds.
- * 2. Download, verify and import backups from connected sync accounts.
- * 3. Upload the updated backup blob.
- */
-export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
-  const providers = await ws.db.iter(Stores.backupProviders).toArray();
-  logger.trace("got backup providers", providers);
-  const backupJson = await exportBackup(ws);
-  const backupConfig = await provideBackupState(ws);
-  const encBackup = await encryptBackup(backupConfig, backupJson);
+interface BackupForProviderArgs {
+  backupConfig: WalletBackupConfState;
+  provider: BackupProviderRecord;
+  currentBackupHash: ArrayBuffer;
+  encBackup: ArrayBuffer;
+  backupJson: WalletBackupContentV1;
 
-  const currentBackupHash = hash(encBackup);
+  /**
+   * Should we attempt one more upload after trying
+   * to pay?
+   */
+  retryAfterPayment: boolean;
+}
 
-  for (const provider of providers) {
-    const accountKeyPair = deriveAccountKeyPair(backupConfig, provider.baseUrl);
-    logger.trace(`trying to upload backup to ${provider.baseUrl}`);
+async function runBackupCycleForProvider(
+  ws: InternalWalletState,
+  args: BackupForProviderArgs,
+): Promise<void> {
+  const {
+    backupConfig,
+    provider,
+    currentBackupHash,
+    encBackup,
+    backupJson,
+  } = args;
+  const accountKeyPair = deriveAccountKeyPair(backupConfig, provider.baseUrl);
+  logger.trace(`trying to upload backup to ${provider.baseUrl}`);
+
+  const syncSig = await ws.cryptoApi.makeSyncSignature({
+    newHash: encodeCrock(currentBackupHash),
+    oldHash: provider.lastBackupHash,
+    accountPriv: encodeCrock(accountKeyPair.eddsaPriv),
+  });
 
-    const syncSig = await ws.cryptoApi.makeSyncSignature({
-      newHash: encodeCrock(currentBackupHash),
-      oldHash: provider.lastBackupHash,
-      accountPriv: encodeCrock(accountKeyPair.eddsaPriv),
-    });
+  logger.trace(`sync signature is ${syncSig}`);
 
-    logger.trace(`sync signature is ${syncSig}`);
+  const accountBackupUrl = new URL(
+    `/backups/${encodeCrock(accountKeyPair.eddsaPub)}`,
+    provider.baseUrl,
+  );
 
-    const accountBackupUrl = new URL(
-      `/backups/${encodeCrock(accountKeyPair.eddsaPub)}`,
-      provider.baseUrl,
-    );
+  const resp = await ws.http.fetch(accountBackupUrl.href, {
+    method: "POST",
+    body: encBackup,
+    headers: {
+      "content-type": "application/octet-stream",
+      "sync-signature": syncSig,
+      "if-none-match": encodeCrock(currentBackupHash),
+      ...(provider.lastBackupHash
+        ? {
+            "if-match": provider.lastBackupHash,
+          }
+        : {}),
+    },
+  });
 
-    const resp = await ws.http.fetch(accountBackupUrl.href, {
-      method: "POST",
-      body: encBackup,
-      headers: {
-        "content-type": "application/octet-stream",
-        "sync-signature": syncSig,
-        "if-none-match": encodeCrock(currentBackupHash),
-        ...(provider.lastBackupHash
-          ? {
-              "if-match": provider.lastBackupHash,
-            }
-          : {}),
-      },
-    });
+  logger.trace(`sync response status: ${resp.status}`);
 
-    logger.trace(`sync response status: ${resp.status}`);
+  if (resp.status === HttpResponseStatus.PaymentRequired) {
+    logger.trace("payment required for backup");
+    logger.trace(`headers: ${j2s(resp.headers)}`);
+    const talerUri = resp.headers.get("taler");
+    if (!talerUri) {
+      throw Error("no taler URI available to pay provider");
+    }
+    const res = await preparePayForUri(ws, talerUri);
+    let proposalId = res.proposalId;
+    let doPay: boolean = false;
+    switch (res.status) {
+      case PreparePayResultType.InsufficientBalance:
+        // FIXME: record in provider state!
+        logger.warn("insufficient balance to pay for backup provider");
+        proposalId = res.proposalId;
+        break;
+      case PreparePayResultType.PaymentPossible:
+        doPay = true;
+        break;
+      case PreparePayResultType.AlreadyConfirmed:
+        break;
+    }
 
-    if (resp.status === HttpResponseStatus.PaymentRequired) {
-      logger.trace("payment required for backup");
-      logger.trace(`headers: ${j2s(resp.headers)}`);
-      const talerUri = resp.headers.get("taler");
-      if (!talerUri) {
-        throw Error("no taler URI available to pay provider");
-      }
-      const res = await preparePayForUri(ws, talerUri);
-      let proposalId: string | undefined;
-      switch (res.status) {
-        case PreparePayResultType.InsufficientBalance:
-          // FIXME: record in provider state!
-          logger.warn("insufficient balance to pay for backup provider");
-          break;
-        case PreparePayResultType.PaymentPossible:
-        case PreparePayResultType.AlreadyConfirmed:
-          proposalId = res.proposalId;
-          break;
-      }
-      if (!proposalId) {
-        continue;
-      }
-      const p = proposalId;
-      await ws.db.runWithWriteTransaction(
-        [Stores.backupProviders],
-        async (tx) => {
-          const provRec = await tx.get(
-            Stores.backupProviders,
-            provider.baseUrl,
-          );
-          checkDbInvariant(!!provRec);
-          const ids = new Set(provRec.paymentProposalIds);
-          ids.add(p);
-          provRec.paymentProposalIds = Array.from(ids);
-          await tx.put(Stores.backupProviders, provRec);
-        },
-      );
+    // FIXME: check if the provider is overcharging us!
+
+    await ws.db.runWithWriteTransaction(
+      [Stores.backupProviders],
+      async (tx) => {
+        const provRec = await tx.get(Stores.backupProviders, provider.baseUrl);
+        checkDbInvariant(!!provRec);
+        const ids = new Set(provRec.paymentProposalIds);
+        ids.add(proposalId);
+        provRec.paymentProposalIds = Array.from(ids).sort();
+        provRec.currentPaymentProposalId = proposalId;
+        await tx.put(Stores.backupProviders, provRec);
+      },
+    );
+
+    if (doPay) {
       const confirmRes = await confirmPay(ws, proposalId);
       switch (confirmRes.type) {
         case ConfirmPayResultType.Pending:
@@ -310,55 +331,41 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
           break;
       }
     }
-    if (resp.status === HttpResponseStatus.NoContent) {
-      await ws.db.runWithWriteTransaction(
-        [Stores.backupProviders],
-        async (tx) => {
-          const prov = await tx.get(Stores.backupProviders, provider.baseUrl);
-          if (!prov) {
-            return;
-          }
-          prov.lastBackupHash = encodeCrock(currentBackupHash);
-          prov.lastBackupTimestamp = getTimestampNow();
-          prov.lastBackupClock =
-            backupJson.clocks[backupJson.current_device_id];
-          prov.lastError = undefined;
-          await tx.put(Stores.backupProviders, prov);
-        },
-      );
-      continue;
-    }
-    if (resp.status === HttpResponseStatus.Conflict) {
-      logger.info("conflicting backup found");
-      const backupEnc = new Uint8Array(await resp.bytes());
-      const backupConfig = await provideBackupState(ws);
-      const blob = await decryptBackup(backupConfig, backupEnc);
-      const cryptoData = await computeBackupCryptoData(ws.cryptoApi, blob);
-      await importBackup(ws, blob, cryptoData);
-      await ws.db.runWithWriteTransaction(
-        [Stores.backupProviders],
-        async (tx) => {
-          const prov = await tx.get(Stores.backupProviders, provider.baseUrl);
-          if (!prov) {
-            return;
-          }
-          prov.lastBackupHash = encodeCrock(hash(backupEnc));
-          prov.lastBackupClock = blob.clocks[blob.current_device_id];
-          prov.lastBackupTimestamp = getTimestampNow();
-          prov.lastError = undefined;
-          await tx.put(Stores.backupProviders, prov);
-        },
-      );
-      logger.info("processed existing backup");
-      continue;
-    }
 
-    // Some other response that we did not expect!
+    if (args.retryAfterPayment) {
+      await runBackupCycleForProvider(ws, {
+        ...args,
+        retryAfterPayment: false,
+      });
+    }
+    return;
+  }
 
-    logger.error("parsing error response");
+  if (resp.status === HttpResponseStatus.NoContent) {
+    await ws.db.runWithWriteTransaction(
+      [Stores.backupProviders],
+      async (tx) => {
+        const prov = await tx.get(Stores.backupProviders, provider.baseUrl);
+        if (!prov) {
+          return;
+        }
+        prov.lastBackupHash = encodeCrock(currentBackupHash);
+        prov.lastBackupTimestamp = getTimestampNow();
+        prov.lastBackupClock = backupJson.clocks[backupJson.current_device_id];
+        prov.lastError = undefined;
+        await tx.put(Stores.backupProviders, prov);
+      },
+    );
+    return;
+  }
 
-    const err = await readTalerErrorResponse(resp);
-    logger.error(`got error response from backup provider: ${j2s(err)}`);
+  if (resp.status === HttpResponseStatus.Conflict) {
+    logger.info("conflicting backup found");
+    const backupEnc = new Uint8Array(await resp.bytes());
+    const backupConfig = await provideBackupState(ws);
+    const blob = await decryptBackup(backupConfig, backupEnc);
+    const cryptoData = await computeBackupCryptoData(ws.cryptoApi, blob);
+    await importBackup(ws, blob, cryptoData);
     await ws.db.runWithWriteTransaction(
       [Stores.backupProviders],
       async (tx) => {
@@ -366,9 +373,58 @@ export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
         if (!prov) {
           return;
         }
-        prov.lastError = err;
+        prov.lastBackupHash = encodeCrock(hash(backupEnc));
+        prov.lastBackupClock = blob.clocks[blob.current_device_id];
+        prov.lastBackupTimestamp = getTimestampNow();
+        prov.lastError = undefined;
+        await tx.put(Stores.backupProviders, prov);
       },
     );
+    logger.info("processed existing backup");
+    return;
+  }
+
+  // Some other response that we did not expect!
+
+  logger.error("parsing error response");
+
+  const err = await readTalerErrorResponse(resp);
+  logger.error(`got error response from backup provider: ${j2s(err)}`);
+  await ws.db.runWithWriteTransaction([Stores.backupProviders], async (tx) => {
+    const prov = await tx.get(Stores.backupProviders, provider.baseUrl);
+    if (!prov) {
+      return;
+    }
+    prov.lastError = err;
+    await tx.put(Stores.backupProviders, prov);
+  });
+}
+
+/**
+ * Do one backup cycle that consists of:
+ * 1. Exporting a backup and try to upload it.
+ *    Stop if this step succeeds.
+ * 2. Download, verify and import backups from connected sync accounts.
+ * 3. Upload the updated backup blob.
+ */
+export async function runBackupCycle(ws: InternalWalletState): Promise<void> {
+  const providers = await ws.db.iter(Stores.backupProviders).toArray();
+  logger.trace("got backup providers", providers);
+  const backupJson = await exportBackup(ws);
+  const backupConfig = await provideBackupState(ws);
+  const encBackup = await encryptBackup(backupConfig, backupJson);
+
+  const currentBackupHash = hash(encBackup);
+
+  for (const provider of providers) {
+    await runBackupCycleForProvider(ws, {
+      provider,
+      backupJson,
+      backupConfig,
+      encBackup,
+      currentBackupHash,
+      retryAfterPayment: true,
+    });
   }
 }
 
@@ -462,8 +518,15 @@ export interface ProviderInfo {
   lastRemoteClock?: number;
   lastBackupTimestamp?: Timestamp;
   paymentProposalIds: string[];
+  paymentStatus: ProviderPaymentStatus;
 }
 
+export type ProviderPaymentStatus =
+  | ProviderPaymentPaid
+  | ProviderPaymentInsufficientBalance
+  | ProviderPaymentUnpaid
+  | ProviderPaymentPending;
+
 export interface BackupInfo {
   walletRootPub: string;
   deviceId: string;
@@ -483,6 +546,71 @@ export async function importBackupPlain(
   await importBackup(ws, blob, cryptoData);
 }
 
+export enum ProviderPaymentType {
+  Unpaid = "unpaid",
+  Pending = "pending",
+  InsufficientBalance = "insufficient-balance",
+  Paid = "paid",
+}
+
+export interface ProviderPaymentUnpaid {
+  type: ProviderPaymentType.Unpaid;
+}
+
+export interface ProviderPaymentInsufficientBalance {
+  type: ProviderPaymentType.InsufficientBalance;
+}
+
+export interface ProviderPaymentPending {
+  type: ProviderPaymentType.Pending;
+}
+
+export interface ProviderPaymentPaid {
+  type: ProviderPaymentType.Paid;
+  paidUntil: Timestamp;
+}
+
+async function getProviderPaymentInfo(
+  ws: InternalWalletState,
+  provider: BackupProviderRecord,
+): Promise<ProviderPaymentStatus> {
+  if (!provider.currentPaymentProposalId) {
+    return {
+      type: ProviderPaymentType.Unpaid,
+    };
+  }
+  const status = await checkPaymentByProposalId(
+    ws,
+    provider.currentPaymentProposalId,
+  );
+  if (status.status === PreparePayResultType.InsufficientBalance) {
+    return {
+      type: ProviderPaymentType.InsufficientBalance,
+    };
+  }
+  if (status.status === PreparePayResultType.PaymentPossible) {
+    return {
+      type: ProviderPaymentType.Pending,
+    };
+  }
+  if (status.status === PreparePayResultType.AlreadyConfirmed) {
+    if (status.paid) {
+      return {
+        type: ProviderPaymentType.Paid,
+        paidUntil: timestampAddDuration(
+          status.contractTerms.timestamp,
+          durationFromSpec({ years: 1 }),
+        ),
+      };
+    } else {
+      return {
+        type: ProviderPaymentType.Pending,
+      };
+    }
+  }
+  throw Error("not reached");
+}
+
 /**
  * Get information about the current state of wallet backups.
  */
@@ -490,19 +618,24 @@ export async function getBackupInfo(
   ws: InternalWalletState,
 ): Promise<BackupInfo> {
   const backupConfig = await provideBackupState(ws);
-  const providers = await ws.db.iter(Stores.backupProviders).toArray();
-  return {
-    deviceId: backupConfig.deviceId,
-    lastLocalClock: backupConfig.clocks[backupConfig.deviceId],
-    walletRootPub: backupConfig.walletRootPub,
-    providers: providers.map((x) => ({
+  const providerRecords = await ws.db.iter(Stores.backupProviders).toArray();
+  const providers: ProviderInfo[] = [];
+  for (const x of providerRecords) {
+    providers.push({
       active: x.active,
       lastRemoteClock: x.lastBackupClock,
       syncProviderBaseUrl: x.baseUrl,
       lastBackupTimestamp: x.lastBackupTimestamp,
       paymentProposalIds: x.paymentProposalIds,
       lastError: x.lastError,
-    })),
+      paymentStatus: await getProviderPaymentInfo(ws, x),
+    });
+  }
+  return {
+    deviceId: backupConfig.deviceId,
+    lastLocalClock: backupConfig.clocks[backupConfig.deviceId],
+    walletRootPub: backupConfig.walletRootPub,
+    providers,
   };
 }
 
diff --git a/packages/taler-wallet-core/src/operations/pay.ts b/packages/taler-wallet-core/src/operations/pay.ts
index cccbb3cac..03bf9e119 100644
--- a/packages/taler-wallet-core/src/operations/pay.ts
+++ b/packages/taler-wallet-core/src/operations/pay.ts
@@ -1150,36 +1150,11 @@ async function submitPay(
   };
 }
 
-/**
- * Check if a payment for the given taler://pay/ URI is possible.
- *
- * If the payment is possible, the signature are already generated but not
- * yet send to the merchant.
- */
-export async function preparePayForUri(
+export async function checkPaymentByProposalId(
   ws: InternalWalletState,
-  talerPayUri: string,
+  proposalId: string,
+  sessionId?: string,
 ): Promise<PreparePayResult> {
-  const uriResult = parsePayUri(talerPayUri);
-
-  if (!uriResult) {
-    throw OperationFailedError.fromCode(
-      TalerErrorCode.WALLET_INVALID_TALER_PAY_URI,
-      `invalid taler://pay URI (${talerPayUri})`,
-      {
-        talerPayUri,
-      },
-    );
-  }
-
-  let proposalId = await startDownloadProposal(
-    ws,
-    uriResult.merchantBaseUrl,
-    uriResult.orderId,
-    uriResult.sessionId,
-    uriResult.claimToken,
-  );
-
   let proposal = await ws.db.get(Stores.proposals, proposalId);
   if (!proposal) {
     throw Error(`could not get proposal ${proposalId}`);
@@ -1238,7 +1213,7 @@ export async function preparePayForUri(
     };
   }
 
-  if (purchase.lastSessionId !== uriResult.sessionId) {
+  if (purchase.lastSessionId !== sessionId) {
     logger.trace(
       "automatically re-submitting payment with different session ID",
     );
@@ -1247,7 +1222,7 @@ export async function preparePayForUri(
       if (!p) {
         return;
       }
-      p.lastSessionId = uriResult.sessionId;
+      p.lastSessionId = sessionId;
       await tx.put(Stores.purchases, p);
     });
     const r = await guardOperationException(
@@ -1293,6 +1268,39 @@ export async function preparePayForUri(
 }
 
 /**
+ * Check if a payment for the given taler://pay/ URI is possible.
+ *
+ * If the payment is possible, the signature are already generated but not
+ * yet send to the merchant.
+ */
+export async function preparePayForUri(
+  ws: InternalWalletState,
+  talerPayUri: string,
+): Promise<PreparePayResult> {
+  const uriResult = parsePayUri(talerPayUri);
+
+  if (!uriResult) {
+    throw OperationFailedError.fromCode(
+      TalerErrorCode.WALLET_INVALID_TALER_PAY_URI,
+      `invalid taler://pay URI (${talerPayUri})`,
+      {
+        talerPayUri,
+      },
+    );
+  }
+
+  let proposalId = await startDownloadProposal(
+    ws,
+    uriResult.merchantBaseUrl,
+    uriResult.orderId,
+    uriResult.sessionId,
+    uriResult.claimToken,
+  );
+
+  return checkPaymentByProposalId(ws, proposalId, uriResult.sessionId);
+}
+
+/**
  * Generate deposit permissions for a purchase.
  *
  * Accesses the database and the crypto worker.