/* This file is part of TALER (C) 2016 GNUnet e.V. TALER is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. TALER is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with TALER; see the file COPYING. If not, see */ /** * @fileoverview * Query helpers for IndexedDB databases. * * @author Florian Dold */ /** * Imports. */ import { IDBCursor, IDBDatabase, IDBFactory, IDBKeyPath, IDBKeyRange, IDBRequest, IDBTransaction, IDBTransactionMode, IDBValidKey, IDBVersionChangeEvent, } from "@gnu-taler/idb-bridge"; import { CancellationToken, Codec, Logger, openPromise, } from "@gnu-taler/taler-util"; const logger = new Logger("query.ts"); /** * Exception that should be thrown by client code to abort a transaction. */ export const TransactionAbort = Symbol("transaction_abort"); /** * Options for an index. */ export interface IndexOptions { /** * If true and the path resolves to an array, create an index entry for * each member of the array (instead of one index entry containing the full array). * * Defaults to false. */ multiEntry?: boolean; /** * Database version that this store was added in, or * undefined if added in the first version. */ versionAdded?: number; /** * Does this index enforce unique keys? * * Defaults to false. */ unique?: boolean; } function requestToPromise(req: IDBRequest): Promise { const stack = Error("Failed request was started here."); return new Promise((resolve, reject) => { req.onsuccess = () => { resolve(req.result); }; req.onerror = () => { console.error("error in DB request", req.error); reject(req.error); console.error("Request failed:", stack); }; }); } type CursorResult = CursorEmptyResult | CursorValueResult; interface CursorEmptyResult { hasValue: false; } interface CursorValueResult { hasValue: true; value: T; } class TransactionAbortedError extends Error { constructor(m: string) { super(m); // Set the prototype explicitly. Object.setPrototypeOf(this, TransactionAbortedError.prototype); } } class ResultStream { private currentPromise: Promise; private gotCursorEnd = false; private awaitingResult = false; constructor(private req: IDBRequest) { this.awaitingResult = true; let p = openPromise(); this.currentPromise = p.promise; req.onsuccess = () => { if (!this.awaitingResult) { throw Error("BUG: invariant violated"); } const cursor = req.result; if (cursor) { this.awaitingResult = false; p.resolve(); p = openPromise(); this.currentPromise = p.promise; } else { this.gotCursorEnd = true; p.resolve(); } }; req.onerror = () => { p.reject(req.error); }; } async toArray(): Promise { const arr: T[] = []; while (true) { const x = await this.next(); if (x.hasValue) { arr.push(x.value); } else { break; } } return arr; } async map(f: (x: T) => R): Promise { const arr: R[] = []; while (true) { const x = await this.next(); if (x.hasValue) { arr.push(f(x.value)); } else { break; } } return arr; } async mapAsync(f: (x: T) => Promise): Promise { const arr: R[] = []; while (true) { const x = await this.next(); if (x.hasValue) { arr.push(await f(x.value)); } else { break; } } return arr; } async forEachAsync(f: (x: T) => Promise): Promise { while (true) { const x = await this.next(); if (x.hasValue) { await f(x.value); } else { break; } } } async forEach(f: (x: T) => void): Promise { while (true) { const x = await this.next(); if (x.hasValue) { f(x.value); } else { break; } } } async filter(f: (x: T) => boolean): Promise { const arr: T[] = []; while (true) { const x = await this.next(); if (x.hasValue) { if (f(x.value)) { arr.push(x.value); } } else { break; } } return arr; } async next(): Promise> { if (this.gotCursorEnd) { return { hasValue: false }; } if (!this.awaitingResult) { const cursor: IDBCursor | undefined = this.req.result; if (!cursor) { throw Error("assertion failed"); } this.awaitingResult = true; cursor.continue(); } await this.currentPromise; if (this.gotCursorEnd) { return { hasValue: false }; } const cursor = this.req.result; if (!cursor) { throw Error("assertion failed"); } return { hasValue: true, value: cursor.value }; } } /** * Return a promise that resolves to the opened IndexedDB database. */ export function openDatabase( idbFactory: IDBFactory, databaseName: string, databaseVersion: number | undefined, onVersionChange: () => void, onUpgradeNeeded: ( db: IDBDatabase, oldVersion: number, newVersion: number, upgradeTransaction: IDBTransaction, ) => void, ): Promise { return new Promise((resolve, reject) => { const req = idbFactory.open(databaseName, databaseVersion); req.onerror = (event) => { // @ts-expect-error reject(new Error(`database opening error`, { cause: req.error })); }; req.onsuccess = (e) => { req.result.onversionchange = (evt: IDBVersionChangeEvent) => { logger.info( `handling versionchange on ${databaseName} from ${evt.oldVersion} to ${evt.newVersion}`, ); req.result.close(); onVersionChange(); }; resolve(req.result); }; req.onupgradeneeded = (e) => { const db = req.result; const newVersion = e.newVersion; if (!newVersion) { // @ts-expect-error throw Error("upgrade needed, but new version unknown", { cause: req.error, }); } const transaction = req.transaction; if (!transaction) { // @ts-expect-error throw Error("no transaction handle available in upgrade handler", { cause: req.error, }); } logger.info( `handling upgradeneeded event on ${databaseName} from ${e.oldVersion} to ${e.newVersion}`, ); onUpgradeNeeded(db, e.oldVersion, newVersion, transaction); }; }); } export interface IndexDescriptor { name: string; keyPath: IDBKeyPath | IDBKeyPath[]; multiEntry?: boolean; unique?: boolean; versionAdded?: number; } export interface StoreDescriptor { _dummy: undefined & RecordType; keyPath?: IDBKeyPath | IDBKeyPath[]; autoIncrement?: boolean; /** * Database version that this store was added in, or * undefined if added in the first version. */ versionAdded?: number; } export interface StoreOptions { keyPath?: IDBKeyPath | IDBKeyPath[]; autoIncrement?: boolean; /** * First minor database version that this store was added in, or * undefined if added in the first version. */ versionAdded?: number; } export function describeContents( options: StoreOptions, ): StoreDescriptor { return { keyPath: options.keyPath, _dummy: undefined as any, autoIncrement: options.autoIncrement, versionAdded: options.versionAdded, }; } export function describeIndex( name: string, keyPath: IDBKeyPath | IDBKeyPath[], options: IndexOptions = {}, ): IndexDescriptor { return { keyPath, name, multiEntry: options.multiEntry, unique: options.unique, versionAdded: options.versionAdded, }; } interface IndexReadOnlyAccessor { iter(query?: IDBKeyRange | IDBValidKey): ResultStream; get(query: IDBValidKey): Promise; getAll( query?: IDBKeyRange | IDBValidKey, count?: number, ): Promise; getAllKeys( query?: IDBKeyRange | IDBValidKey, count?: number, ): Promise; count(query?: IDBValidKey): Promise; } type GetIndexReadOnlyAccess = { [P in keyof IndexMap]: IndexReadOnlyAccessor; }; interface IndexReadWriteAccessor { iter(query: IDBKeyRange | IDBValidKey): ResultStream; get(query: IDBValidKey): Promise; getAll( query?: IDBKeyRange | IDBValidKey, count?: number, ): Promise; getAllKeys( query?: IDBKeyRange | IDBValidKey, count?: number, ): Promise; count(query?: IDBValidKey): Promise; } type GetIndexReadWriteAccess = { [P in keyof IndexMap]: IndexReadWriteAccessor; }; export interface StoreReadOnlyAccessor { get(key: IDBValidKey): Promise; getAll( query?: IDBKeyRange | IDBValidKey, count?: number, ): Promise; iter(query?: IDBValidKey): ResultStream; indexes: GetIndexReadOnlyAccess; } export interface InsertResponse { /** * Key of the newly inserted (via put/add) record. */ key: IDBValidKey; } export interface StoreReadWriteAccessor { get(key: IDBValidKey): Promise; getAll( query?: IDBKeyRange | IDBValidKey, count?: number, ): Promise; iter(query?: IDBValidKey): ResultStream; put(r: RecordType, key?: IDBValidKey): Promise; add(r: RecordType, key?: IDBValidKey): Promise; delete(key: IDBValidKey): Promise; indexes: GetIndexReadWriteAccess; } export interface StoreWithIndexes< StoreName extends string, RecordType, IndexMap, > { storeName: StoreName; store: StoreDescriptor; indexMap: IndexMap; /** * Type marker symbol, to check that the descriptor * has been created through the right function. */ mark: Symbol; } const storeWithIndexesSymbol = Symbol("StoreWithIndexesMark"); export function describeStore( name: StoreName, s: StoreDescriptor, m: IndexMap, ): StoreWithIndexes { return { storeName: name, store: s, indexMap: m, mark: storeWithIndexesSymbol, }; } export function describeStoreV2< StoreName extends string, RecordType, IndexMap extends { [x: string]: IndexDescriptor } = {}, >(args: { storeName: StoreName; recordCodec: Codec; keyPath?: IDBKeyPath | IDBKeyPath[]; autoIncrement?: boolean; /** * Database version that this store was added in, or * undefined if added in the first version. */ versionAdded?: number; indexes?: IndexMap; }): StoreWithIndexes { return { storeName: args.storeName, store: { _dummy: undefined as any, autoIncrement: args.autoIncrement, keyPath: args.keyPath, versionAdded: args.versionAdded, }, indexMap: args.indexes ?? ({} as IndexMap), mark: storeWithIndexesSymbol, }; } type KeyPathComponents = string | number; /** * Follow a key path (dot-separated) in an object. */ type DerefKeyPath = P extends `${infer PX extends keyof T & KeyPathComponents}` ? T[PX] : P extends `${infer P0 extends keyof T & KeyPathComponents}.${infer Rest}` ? DerefKeyPath : unknown; /** * Return a path if it is a valid dot-separate path to an object. * Otherwise, return "never". */ type ValidateKeyPath = P extends `${infer PX extends keyof T & KeyPathComponents}` ? PX : P extends `${infer P0 extends keyof T & KeyPathComponents}.${infer Rest}` ? `${P0}.${ValidateKeyPath}` : never; // function foo( // x: T, // p: P extends ValidateKeyPath ? P : never, // ): void {} // foo({x: [0,1,2]}, "x.0"); export type StoreNames = StoreMap extends { [P in keyof StoreMap]: StoreWithIndexes; } ? keyof StoreMap : unknown; export type DbReadWriteTransaction< StoreMap, StoresArr extends Array>, > = StoreMap extends { [P in string]: StoreWithIndexes; } ? { [X in StoresArr[number] & keyof StoreMap]: StoreMap[X] extends StoreWithIndexes< infer _StoreName, infer RecordType, infer IndexMap > ? StoreReadWriteAccessor : unknown; } : never; export type DbReadOnlyTransaction< StoreMap, StoresArr extends Array>, > = StoreMap extends { [P in string]: StoreWithIndexes; } ? { [X in StoresArr[number] & keyof StoreMap]: StoreMap[X] extends StoreWithIndexes< infer _StoreName, infer RecordType, infer IndexMap > ? StoreReadOnlyAccessor : unknown; } : never; /** * Convert the type of an array to a union of the contents. * * Example: * Input ["foo", "bar"] * Output "foo" | "bar" */ export type UnionFromArray = Arr extends { [X in keyof Arr]: Arr[X] & string; } ? Arr[keyof Arr & number] : unknown; function runTx( tx: IDBTransaction, arg: Arg, f: (t: Arg, t2: IDBTransaction) => Promise, triggerContext: InternalTriggerContext, ): Promise { const stack = Error("Failed transaction was started here."); return new Promise((resolve, reject) => { let funResult: any = undefined; let gotFunResult = false; let transactionException: any = undefined; tx.oncomplete = () => { // This is a fatal error: The transaction completed *before* // the transaction function returned. Likely, the transaction // function waited on a promise that is *not* resolved in the // microtask queue, thus triggering the auto-commit behavior. // Unfortunately, the auto-commit behavior of IDB can't be switched // of. There are some proposals to add this functionality in the future. if (!gotFunResult) { const msg = "BUG: transaction closed before transaction function returned"; logger.error(msg); logger.error(`${stack.stack}`); reject(Error(msg)); } triggerContext.handleAfterCommit(); resolve(funResult); }; tx.onerror = () => { logger.error("error in transaction"); logger.error(`${stack.stack}`); }; tx.onabort = () => { let msg: string; if (tx.error) { msg = `Transaction aborted (transaction error): ${tx.error}`; } else if (transactionException !== undefined) { msg = `Transaction aborted (exception thrown): ${transactionException}`; } else { msg = "Transaction aborted (no DB error)"; } logger.error(msg); logger.error(`${stack.stack}`); reject(new TransactionAbortedError(msg)); }; const resP = Promise.resolve().then(() => f(arg, tx)); resP .then((result) => { gotFunResult = true; funResult = result; }) .catch((e) => { if (e == TransactionAbort) { logger.trace("aborting transaction"); } else { transactionException = e; console.error("Transaction failed:", e); console.error(stack); tx.abort(); } }) .catch((e) => { console.error("fatal: aborting transaction failed", e); }); }); } function makeReadContext( tx: IDBTransaction, storePick: { [n: string]: StoreWithIndexes }, triggerContext: InternalTriggerContext, ): any { const ctx: { [s: string]: StoreReadOnlyAccessor } = {}; for (const storeAlias in storePick) { const indexes: { [s: string]: IndexReadOnlyAccessor } = {}; const swi = storePick[storeAlias]; const storeName = swi.storeName; for (const indexAlias in storePick[storeAlias].indexMap) { const indexDescriptor: IndexDescriptor = storePick[storeAlias].indexMap[indexAlias]; const indexName = indexDescriptor.name; indexes[indexAlias] = { get(key) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).index(indexName).get(key); return requestToPromise(req); }, iter(query) { triggerContext.storesAccessed.add(storeName); const req = tx .objectStore(storeName) .index(indexName) .openCursor(query); return new ResultStream(req); }, getAll(query, count) { triggerContext.storesAccessed.add(storeName); const req = tx .objectStore(storeName) .index(indexName) .getAll(query, count); return requestToPromise(req); }, getAllKeys(query, count) { triggerContext.storesAccessed.add(storeName); const req = tx .objectStore(storeName) .index(indexName) .getAllKeys(query, count); return requestToPromise(req); }, count(query) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).index(indexName).count(query); return requestToPromise(req); }, }; } ctx[storeAlias] = { indexes, get(key) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).get(key); return requestToPromise(req); }, getAll(query, count) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).getAll(query, count); return requestToPromise(req); }, iter(query) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).openCursor(query); return new ResultStream(req); }, }; } return ctx; } function makeWriteContext( tx: IDBTransaction, storePick: { [n: string]: StoreWithIndexes }, triggerContext: InternalTriggerContext, ): any { const ctx: { [s: string]: StoreReadWriteAccessor } = {}; for (const storeAlias in storePick) { const indexes: { [s: string]: IndexReadWriteAccessor } = {}; const swi = storePick[storeAlias]; const storeName = swi.storeName; for (const indexAlias in storePick[storeAlias].indexMap) { const indexDescriptor: IndexDescriptor = storePick[storeAlias].indexMap[indexAlias]; const indexName = indexDescriptor.name; indexes[indexAlias] = { get(key) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).index(indexName).get(key); return requestToPromise(req); }, iter(query) { triggerContext.storesAccessed.add(storeName); const req = tx .objectStore(storeName) .index(indexName) .openCursor(query); return new ResultStream(req); }, getAll(query, count) { triggerContext.storesAccessed.add(storeName); const req = tx .objectStore(storeName) .index(indexName) .getAll(query, count); return requestToPromise(req); }, getAllKeys(query, count) { triggerContext.storesAccessed.add(storeName); const req = tx .objectStore(storeName) .index(indexName) .getAllKeys(query, count); return requestToPromise(req); }, count(query) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).index(indexName).count(query); return requestToPromise(req); }, }; } ctx[storeAlias] = { indexes, get(key) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).get(key); return requestToPromise(req); }, getAll(query, count) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).getAll(query, count); return requestToPromise(req); }, iter(query) { triggerContext.storesAccessed.add(storeName); const req = tx.objectStore(storeName).openCursor(query); return new ResultStream(req); }, async add(r, k) { triggerContext.storesAccessed.add(storeName); triggerContext.storesModified.add(storeName); const req = tx.objectStore(storeName).add(r, k); const key = await requestToPromise(req); return { key: key, }; }, async put(r, k) { triggerContext.storesAccessed.add(storeName); triggerContext.storesModified.add(storeName); const req = tx.objectStore(storeName).put(r, k); const key = await requestToPromise(req); return { key: key, }; }, delete(k) { triggerContext.storesAccessed.add(storeName); triggerContext.storesModified.add(storeName); const req = tx.objectStore(storeName).delete(k); return requestToPromise(req); }, }; } return ctx; } export interface DbAccess { idbHandle(): IDBDatabase; runAllStoresReadWriteTx( options: { label?: string; }, txf: ( tx: DbReadWriteTransaction>>, ) => Promise, ): Promise; runAllStoresReadOnlyTx( options: { label?: string; }, txf: ( tx: DbReadOnlyTransaction>>, ) => Promise, ): Promise; runReadWriteTx>>( opts: { storeNames: StoreNameArray; label?: string; }, txf: (tx: DbReadWriteTransaction) => Promise, ): Promise; runReadOnlyTx>>( opts: { storeNames: StoreNameArray; label?: string; }, txf: (tx: DbReadOnlyTransaction) => Promise, ): Promise; } export interface AfterCommitInfo { mode: IDBTransactionMode; scope: Set; accessedStores: Set; modifiedStores: Set; } export interface TriggerSpec { /** * Trigger run after every successful commit, run outside of the transaction. */ afterCommit?: (info: AfterCommitInfo) => void; // onRead(store, value) // initState () => State // beforeCommit? (tx: Transaction, s: State | undefined) => Promise; } class InternalTriggerContext { storesScope: Set; storesAccessed: Set = new Set(); storesModified: Set = new Set(); constructor( private triggerSpec: TriggerSpec, private mode: IDBTransactionMode, scope: string[], ) { this.storesScope = new Set(scope); } handleAfterCommit() { if (this.triggerSpec.afterCommit) { this.triggerSpec.afterCommit({ mode: this.mode, accessedStores: this.storesAccessed, modifiedStores: this.storesModified, scope: this.storesScope, }); } } } /** * Type-safe access to a database with a particular store map. * * A store map is the metadata that describes the store. */ export class DbAccessImpl implements DbAccess { constructor( private db: IDBDatabase, private stores: StoreMap, private triggers: TriggerSpec = {}, private cancellationToken: CancellationToken, ) {} idbHandle(): IDBDatabase { return this.db; } runAllStoresReadWriteTx( options: { label?: string; }, txf: ( tx: DbReadWriteTransaction>>, ) => Promise, ): Promise { const accessibleStores: { [x: string]: StoreWithIndexes } = {}; const strStoreNames: string[] = []; for (const sn of Object.keys(this.stores as any)) { const swi = (this.stores as any)[sn] as StoreWithIndexes; strStoreNames.push(swi.storeName); accessibleStores[swi.storeName] = swi; } const mode = "readwrite"; const triggerContext = new InternalTriggerContext( this.triggers, mode, strStoreNames, ); const tx = this.db.transaction(strStoreNames, mode); const writeContext = makeWriteContext(tx, accessibleStores, triggerContext); return runTx(tx, writeContext, txf, triggerContext); } async runAllStoresReadOnlyTx( options: { label?: string; }, txf: ( tx: DbReadOnlyTransaction>>, ) => Promise, ): Promise { const accessibleStores: { [x: string]: StoreWithIndexes } = {}; const strStoreNames: string[] = []; for (const sn of Object.keys(this.stores as any)) { const swi = (this.stores as any)[sn] as StoreWithIndexes; strStoreNames.push(swi.storeName); accessibleStores[swi.storeName] = swi; } const mode = "readonly"; const triggerContext = new InternalTriggerContext( this.triggers, mode, strStoreNames, ); const tx = this.db.transaction(strStoreNames, mode); const writeContext = makeReadContext(tx, accessibleStores, triggerContext); const res = await runTx(tx, writeContext, txf, triggerContext); return res; } async runReadWriteTx>>( opts: { storeNames: StoreNameArray; }, txf: (tx: DbReadWriteTransaction) => Promise, ): Promise { const accessibleStores: { [x: string]: StoreWithIndexes } = {}; const strStoreNames: string[] = []; for (const sn of opts.storeNames) { const swi = (this.stores as any)[sn] as StoreWithIndexes; strStoreNames.push(swi.storeName); accessibleStores[swi.storeName] = swi; } const mode = "readwrite"; const triggerContext = new InternalTriggerContext( this.triggers, mode, strStoreNames, ); const tx = this.db.transaction(strStoreNames, mode); const writeContext = makeWriteContext(tx, accessibleStores, triggerContext); const res = await runTx(tx, writeContext, txf, triggerContext); return res; } runReadOnlyTx>>( opts: { storeNames: StoreNameArray; }, txf: (tx: DbReadOnlyTransaction) => Promise, ): Promise { const accessibleStores: { [x: string]: StoreWithIndexes } = {}; const strStoreNames: string[] = []; for (const sn of opts.storeNames) { const swi = (this.stores as any)[sn] as StoreWithIndexes; strStoreNames.push(swi.storeName); accessibleStores[swi.storeName] = swi; } const mode = "readonly"; const triggerContext = new InternalTriggerContext( this.triggers, mode, strStoreNames, ); const tx = this.db.transaction(strStoreNames, mode); const readContext = makeReadContext(tx, accessibleStores, triggerContext); const res = runTx(tx, readContext, txf, triggerContext); return res; } }