From 1392dc47c6489fca1b3a4c036852873495190c36 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Wed, 10 Mar 2021 17:11:59 +0100 Subject: finish first complete end-to-end backup/sync test --- .../src/operations/backup/import.ts | 48 +-- .../src/operations/backup/index.ts | 405 ++++++++++++++------- 2 files changed, 284 insertions(+), 169 deletions(-) (limited to 'packages/taler-wallet-core/src/operations/backup') diff --git a/packages/taler-wallet-core/src/operations/backup/import.ts b/packages/taler-wallet-core/src/operations/backup/import.ts index fa0819745..416b068e4 100644 --- a/packages/taler-wallet-core/src/operations/backup/import.ts +++ b/packages/taler-wallet-core/src/operations/backup/import.ts @@ -15,68 +15,47 @@ */ import { - Stores, - Amounts, - CoinSourceType, - CoinStatus, - RefundState, AbortStatus, - ProposalStatus, - getTimestampNow, - encodeCrock, - stringToBytes, - getRandomBytes, AmountJson, + Amounts, codecForContractTerms, CoinSource, + CoinSourceType, + CoinStatus, DenominationStatus, DenomSelectionState, ExchangeUpdateStatus, ExchangeWireInfo, + getTimestampNow, PayCoinSelection, ProposalDownload, + ProposalStatus, RefreshReason, RefreshSessionRecord, + RefundState, ReserveBankInfo, ReserveRecordStatus, + Stores, TransactionHandle, WalletContractData, WalletRefundItem, } from "../.."; -import { hash } from "../../crypto/primitives/nacl-fast"; import { - WalletBackupContentV1, - BackupExchange, - BackupCoin, - BackupDenomination, - BackupReserve, - BackupPurchase, - BackupProposal, - BackupRefreshGroup, - BackupBackupProvider, - BackupTip, - BackupRecoupGroup, - BackupWithdrawalGroup, - BackupBackupProviderTerms, - BackupCoinSource, BackupCoinSourceType, - BackupExchangeWireFee, - BackupRefundItem, - BackupRefundState, - BackupProposalStatus, - BackupRefreshOldCoin, - BackupRefreshSession, BackupDenomSel, + BackupProposalStatus, + BackupPurchase, BackupRefreshReason, + BackupRefundState, + WalletBackupContentV1, } from "../../types/backupTypes"; -import { canonicalizeBaseUrl, canonicalJson, j2s } from "../../util/helpers"; +import { j2s } from "../../util/helpers"; import { checkDbInvariant, checkLogicInvariant } from "../../util/invariants"; import { Logger } from "../../util/logging"; import { initRetryInfo } from "../../util/retries"; import { InternalWalletState } from "../state"; import { provideBackupState } from "./state"; - const logger = new Logger("operations/backup/import.ts"); function checkBackupInvariant(b: boolean, m?: string): asserts b { @@ -230,6 +209,9 @@ export async function importBackup( cryptoComp: BackupCryptoPrecomputedData, ): Promise { await provideBackupState(ws); + + logger.info(`importing backup ${j2s(backupBlobArg)}`); + return ws.db.runWithWriteTransaction( [ Stores.config, diff --git a/packages/taler-wallet-core/src/operations/backup/index.ts b/packages/taler-wallet-core/src/operations/backup/index.ts index fd0274219..edc5acc15 100644 --- a/packages/taler-wallet-core/src/operations/backup/index.ts +++ b/packages/taler-wallet-core/src/operations/backup/index.ts @@ -27,7 +27,11 @@ import { InternalWalletState } from "../state"; import { WalletBackupContentV1 } from "../../types/backupTypes"; import { TransactionHandle } from "../../util/query"; -import { ConfigRecord, Stores } from "../../types/dbTypes"; +import { + BackupProviderRecord, + ConfigRecord, + Stores, +} from "../../types/dbTypes"; import { checkDbInvariant, checkLogicInvariant } from "../../util/invariants"; import { codecForAmountString } from "../../util/amounts"; import { @@ -41,7 +45,13 @@ import { stringToBytes, } from "../../crypto/talerCrypto"; import { canonicalizeBaseUrl, canonicalJson, j2s } from "../../util/helpers"; -import { getTimestampNow, Timestamp } from "../../util/time"; +import { + durationAdd, + durationFromSpec, + getTimestampNow, + Timestamp, + timestampAddDuration, +} from "../../util/time"; import { URL } from "../../util/url"; import { AmountString } from "../../types/talerTypes"; import { @@ -70,7 +80,7 @@ import { } from "../../types/walletTypes"; import { CryptoApi } from "../../crypto/workers/cryptoApi"; import { secretbox, secretbox_open } from "../../crypto/primitives/nacl-fast"; -import { confirmPay, preparePayForUri } from "../pay"; +import { checkPaymentByProposalId, confirmPay, preparePayForUri } from "../pay"; import { exportBackup } from "./export"; import { BackupCryptoPrecomputedData, importBackup } from "./import"; import { @@ -79,6 +89,7 @@ import { getWalletBackupState, WalletBackupConfState, } from "./state"; +import { PaymentStatus } from "../../types/transactionsTypes"; const logger = new Logger("operations/backup.ts"); @@ -216,93 +227,103 @@ function deriveBlobSecret(bc: WalletBackupConfState): Uint8Array { ); } -/** - * Do one backup cycle that consists of: - * 1. Exporting a backup and try to upload it. - * Stop if this step succeeds. - * 2. Download, verify and import backups from connected sync accounts. - * 3. Upload the updated backup blob. - */ -export async function runBackupCycle(ws: InternalWalletState): Promise { - const providers = await ws.db.iter(Stores.backupProviders).toArray(); - logger.trace("got backup providers", providers); - const backupJson = await exportBackup(ws); - const backupConfig = await provideBackupState(ws); - const encBackup = await encryptBackup(backupConfig, backupJson); +interface BackupForProviderArgs { + backupConfig: WalletBackupConfState; + provider: BackupProviderRecord; + currentBackupHash: ArrayBuffer; + encBackup: ArrayBuffer; + backupJson: WalletBackupContentV1; - const currentBackupHash = hash(encBackup); + /** + * Should we attempt one more upload after trying + * to pay? + */ + retryAfterPayment: boolean; +} - for (const provider of providers) { - const accountKeyPair = deriveAccountKeyPair(backupConfig, provider.baseUrl); - logger.trace(`trying to upload backup to ${provider.baseUrl}`); +async function runBackupCycleForProvider( + ws: InternalWalletState, + args: BackupForProviderArgs, +): Promise { + const { + backupConfig, + provider, + currentBackupHash, + encBackup, + backupJson, + } = args; + const accountKeyPair = deriveAccountKeyPair(backupConfig, provider.baseUrl); + logger.trace(`trying to upload backup to ${provider.baseUrl}`); + + const syncSig = await ws.cryptoApi.makeSyncSignature({ + newHash: encodeCrock(currentBackupHash), + oldHash: provider.lastBackupHash, + accountPriv: encodeCrock(accountKeyPair.eddsaPriv), + }); - const syncSig = await ws.cryptoApi.makeSyncSignature({ - newHash: encodeCrock(currentBackupHash), - oldHash: provider.lastBackupHash, - accountPriv: encodeCrock(accountKeyPair.eddsaPriv), - }); + logger.trace(`sync signature is ${syncSig}`); - logger.trace(`sync signature is ${syncSig}`); + const accountBackupUrl = new URL( + `/backups/${encodeCrock(accountKeyPair.eddsaPub)}`, + provider.baseUrl, + ); - const accountBackupUrl = new URL( - `/backups/${encodeCrock(accountKeyPair.eddsaPub)}`, - provider.baseUrl, - ); + const resp = await ws.http.fetch(accountBackupUrl.href, { + method: "POST", + body: encBackup, + headers: { + "content-type": "application/octet-stream", + "sync-signature": syncSig, + "if-none-match": encodeCrock(currentBackupHash), + ...(provider.lastBackupHash + ? { + "if-match": provider.lastBackupHash, + } + : {}), + }, + }); - const resp = await ws.http.fetch(accountBackupUrl.href, { - method: "POST", - body: encBackup, - headers: { - "content-type": "application/octet-stream", - "sync-signature": syncSig, - "if-none-match": encodeCrock(currentBackupHash), - ...(provider.lastBackupHash - ? { - "if-match": provider.lastBackupHash, - } - : {}), - }, - }); + logger.trace(`sync response status: ${resp.status}`); - logger.trace(`sync response status: ${resp.status}`); + if (resp.status === HttpResponseStatus.PaymentRequired) { + logger.trace("payment required for backup"); + logger.trace(`headers: ${j2s(resp.headers)}`); + const talerUri = resp.headers.get("taler"); + if (!talerUri) { + throw Error("no taler URI available to pay provider"); + } + const res = await preparePayForUri(ws, talerUri); + let proposalId = res.proposalId; + let doPay: boolean = false; + switch (res.status) { + case PreparePayResultType.InsufficientBalance: + // FIXME: record in provider state! + logger.warn("insufficient balance to pay for backup provider"); + proposalId = res.proposalId; + break; + case PreparePayResultType.PaymentPossible: + doPay = true; + break; + case PreparePayResultType.AlreadyConfirmed: + break; + } - if (resp.status === HttpResponseStatus.PaymentRequired) { - logger.trace("payment required for backup"); - logger.trace(`headers: ${j2s(resp.headers)}`); - const talerUri = resp.headers.get("taler"); - if (!talerUri) { - throw Error("no taler URI available to pay provider"); - } - const res = await preparePayForUri(ws, talerUri); - let proposalId: string | undefined; - switch (res.status) { - case PreparePayResultType.InsufficientBalance: - // FIXME: record in provider state! - logger.warn("insufficient balance to pay for backup provider"); - break; - case PreparePayResultType.PaymentPossible: - case PreparePayResultType.AlreadyConfirmed: - proposalId = res.proposalId; - break; - } - if (!proposalId) { - continue; - } - const p = proposalId; - await ws.db.runWithWriteTransaction( - [Stores.backupProviders], - async (tx) => { - const provRec = await tx.get( - Stores.backupProviders, - provider.baseUrl, - ); - checkDbInvariant(!!provRec); - const ids = new Set(provRec.paymentProposalIds); - ids.add(p); - provRec.paymentProposalIds = Array.from(ids); - await tx.put(Stores.backupProviders, provRec); - }, - ); + // FIXME: check if the provider is overcharging us! + + await ws.db.runWithWriteTransaction( + [Stores.backupProviders], + async (tx) => { + const provRec = await tx.get(Stores.backupProviders, provider.baseUrl); + checkDbInvariant(!!provRec); + const ids = new Set(provRec.paymentProposalIds); + ids.add(proposalId); + provRec.paymentProposalIds = Array.from(ids).sort(); + provRec.currentPaymentProposalId = proposalId; + await tx.put(Stores.backupProviders, provRec); + }, + ); + + if (doPay) { const confirmRes = await confirmPay(ws, proposalId); switch (confirmRes.type) { case ConfirmPayResultType.Pending: @@ -310,55 +331,41 @@ export async function runBackupCycle(ws: InternalWalletState): Promise { break; } } - if (resp.status === HttpResponseStatus.NoContent) { - await ws.db.runWithWriteTransaction( - [Stores.backupProviders], - async (tx) => { - const prov = await tx.get(Stores.backupProviders, provider.baseUrl); - if (!prov) { - return; - } - prov.lastBackupHash = encodeCrock(currentBackupHash); - prov.lastBackupTimestamp = getTimestampNow(); - prov.lastBackupClock = - backupJson.clocks[backupJson.current_device_id]; - prov.lastError = undefined; - await tx.put(Stores.backupProviders, prov); - }, - ); - continue; - } - if (resp.status === HttpResponseStatus.Conflict) { - logger.info("conflicting backup found"); - const backupEnc = new Uint8Array(await resp.bytes()); - const backupConfig = await provideBackupState(ws); - const blob = await decryptBackup(backupConfig, backupEnc); - const cryptoData = await computeBackupCryptoData(ws.cryptoApi, blob); - await importBackup(ws, blob, cryptoData); - await ws.db.runWithWriteTransaction( - [Stores.backupProviders], - async (tx) => { - const prov = await tx.get(Stores.backupProviders, provider.baseUrl); - if (!prov) { - return; - } - prov.lastBackupHash = encodeCrock(hash(backupEnc)); - prov.lastBackupClock = blob.clocks[blob.current_device_id]; - prov.lastBackupTimestamp = getTimestampNow(); - prov.lastError = undefined; - await tx.put(Stores.backupProviders, prov); - }, - ); - logger.info("processed existing backup"); - continue; - } - // Some other response that we did not expect! + if (args.retryAfterPayment) { + await runBackupCycleForProvider(ws, { + ...args, + retryAfterPayment: false, + }); + } + return; + } - logger.error("parsing error response"); + if (resp.status === HttpResponseStatus.NoContent) { + await ws.db.runWithWriteTransaction( + [Stores.backupProviders], + async (tx) => { + const prov = await tx.get(Stores.backupProviders, provider.baseUrl); + if (!prov) { + return; + } + prov.lastBackupHash = encodeCrock(currentBackupHash); + prov.lastBackupTimestamp = getTimestampNow(); + prov.lastBackupClock = backupJson.clocks[backupJson.current_device_id]; + prov.lastError = undefined; + await tx.put(Stores.backupProviders, prov); + }, + ); + return; + } - const err = await readTalerErrorResponse(resp); - logger.error(`got error response from backup provider: ${j2s(err)}`); + if (resp.status === HttpResponseStatus.Conflict) { + logger.info("conflicting backup found"); + const backupEnc = new Uint8Array(await resp.bytes()); + const backupConfig = await provideBackupState(ws); + const blob = await decryptBackup(backupConfig, backupEnc); + const cryptoData = await computeBackupCryptoData(ws.cryptoApi, blob); + await importBackup(ws, blob, cryptoData); await ws.db.runWithWriteTransaction( [Stores.backupProviders], async (tx) => { @@ -366,9 +373,58 @@ export async function runBackupCycle(ws: InternalWalletState): Promise { if (!prov) { return; } - prov.lastError = err; + prov.lastBackupHash = encodeCrock(hash(backupEnc)); + prov.lastBackupClock = blob.clocks[blob.current_device_id]; + prov.lastBackupTimestamp = getTimestampNow(); + prov.lastError = undefined; + await tx.put(Stores.backupProviders, prov); }, ); + logger.info("processed existing backup"); + return; + } + + // Some other response that we did not expect! + + logger.error("parsing error response"); + + const err = await readTalerErrorResponse(resp); + logger.error(`got error response from backup provider: ${j2s(err)}`); + await ws.db.runWithWriteTransaction([Stores.backupProviders], async (tx) => { + const prov = await tx.get(Stores.backupProviders, provider.baseUrl); + if (!prov) { + return; + } + prov.lastError = err; + await tx.put(Stores.backupProviders, prov); + }); +} + +/** + * Do one backup cycle that consists of: + * 1. Exporting a backup and try to upload it. + * Stop if this step succeeds. + * 2. Download, verify and import backups from connected sync accounts. + * 3. Upload the updated backup blob. + */ +export async function runBackupCycle(ws: InternalWalletState): Promise { + const providers = await ws.db.iter(Stores.backupProviders).toArray(); + logger.trace("got backup providers", providers); + const backupJson = await exportBackup(ws); + const backupConfig = await provideBackupState(ws); + const encBackup = await encryptBackup(backupConfig, backupJson); + + const currentBackupHash = hash(encBackup); + + for (const provider of providers) { + await runBackupCycleForProvider(ws, { + provider, + backupJson, + backupConfig, + encBackup, + currentBackupHash, + retryAfterPayment: true, + }); } } @@ -462,8 +518,15 @@ export interface ProviderInfo { lastRemoteClock?: number; lastBackupTimestamp?: Timestamp; paymentProposalIds: string[]; + paymentStatus: ProviderPaymentStatus; } +export type ProviderPaymentStatus = + | ProviderPaymentPaid + | ProviderPaymentInsufficientBalance + | ProviderPaymentUnpaid + | ProviderPaymentPending; + export interface BackupInfo { walletRootPub: string; deviceId: string; @@ -483,6 +546,71 @@ export async function importBackupPlain( await importBackup(ws, blob, cryptoData); } +export enum ProviderPaymentType { + Unpaid = "unpaid", + Pending = "pending", + InsufficientBalance = "insufficient-balance", + Paid = "paid", +} + +export interface ProviderPaymentUnpaid { + type: ProviderPaymentType.Unpaid; +} + +export interface ProviderPaymentInsufficientBalance { + type: ProviderPaymentType.InsufficientBalance; +} + +export interface ProviderPaymentPending { + type: ProviderPaymentType.Pending; +} + +export interface ProviderPaymentPaid { + type: ProviderPaymentType.Paid; + paidUntil: Timestamp; +} + +async function getProviderPaymentInfo( + ws: InternalWalletState, + provider: BackupProviderRecord, +): Promise { + if (!provider.currentPaymentProposalId) { + return { + type: ProviderPaymentType.Unpaid, + }; + } + const status = await checkPaymentByProposalId( + ws, + provider.currentPaymentProposalId, + ); + if (status.status === PreparePayResultType.InsufficientBalance) { + return { + type: ProviderPaymentType.InsufficientBalance, + }; + } + if (status.status === PreparePayResultType.PaymentPossible) { + return { + type: ProviderPaymentType.Pending, + }; + } + if (status.status === PreparePayResultType.AlreadyConfirmed) { + if (status.paid) { + return { + type: ProviderPaymentType.Paid, + paidUntil: timestampAddDuration( + status.contractTerms.timestamp, + durationFromSpec({ years: 1 }), + ), + }; + } else { + return { + type: ProviderPaymentType.Pending, + }; + } + } + throw Error("not reached"); +} + /** * Get information about the current state of wallet backups. */ @@ -490,19 +618,24 @@ export async function getBackupInfo( ws: InternalWalletState, ): Promise { const backupConfig = await provideBackupState(ws); - const providers = await ws.db.iter(Stores.backupProviders).toArray(); - return { - deviceId: backupConfig.deviceId, - lastLocalClock: backupConfig.clocks[backupConfig.deviceId], - walletRootPub: backupConfig.walletRootPub, - providers: providers.map((x) => ({ + const providerRecords = await ws.db.iter(Stores.backupProviders).toArray(); + const providers: ProviderInfo[] = []; + for (const x of providerRecords) { + providers.push({ active: x.active, lastRemoteClock: x.lastBackupClock, syncProviderBaseUrl: x.baseUrl, lastBackupTimestamp: x.lastBackupTimestamp, paymentProposalIds: x.paymentProposalIds, lastError: x.lastError, - })), + paymentStatus: await getProviderPaymentInfo(ws, x), + }); + } + return { + deviceId: backupConfig.deviceId, + lastLocalClock: backupConfig.clocks[backupConfig.deviceId], + walletRootPub: backupConfig.walletRootPub, + providers, }; } -- cgit v1.2.3