diff options
Diffstat (limited to 'packages/idb-bridge/src/SqliteBackend.ts')
| -rw-r--r-- | packages/idb-bridge/src/SqliteBackend.ts | 2329 |
1 files changed, 2329 insertions, 0 deletions
diff --git a/packages/idb-bridge/src/SqliteBackend.ts b/packages/idb-bridge/src/SqliteBackend.ts new file mode 100644 index 000000000..26ed43b0f --- /dev/null +++ b/packages/idb-bridge/src/SqliteBackend.ts @@ -0,0 +1,2329 @@ +/* + Copyright 2023 Taler Systems S.A. + + GNU Taler is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> + */ + +/** + * Imports. + */ +import { AsyncCondition } from "./backend-common.js"; +import { + Backend, + ConnectResult, + DatabaseConnection, + DatabaseTransaction, + IndexGetQuery, + IndexMeta, + ObjectStoreGetQuery, + ObjectStoreMeta, + RecordGetResponse, + RecordStoreRequest, + RecordStoreResponse, + ResultLevel, + StoreLevel, +} from "./backend-interface.js"; +import { BridgeIDBDatabaseInfo, BridgeIDBKeyRange } from "./bridge-idb.js"; +import { + IDBKeyPath, + IDBKeyRange, + IDBTransactionMode, + IDBValidKey, +} from "./idbtypes.js"; +import { + AccessStats, + structuredEncapsulate, + structuredRevive, +} from "./index.js"; +import { ConstraintError, DataError } from "./util/errors.js"; +import { getIndexKeys } from "./util/getIndexKeys.js"; +import { deserializeKey, serializeKey } from "./util/key-storage.js"; +import { makeStoreKeyValue } from "./util/makeStoreKeyValue.js"; +import { + Sqlite3Database, + Sqlite3Interface, + Sqlite3Statement, +} from "./sqlite3-interface.js"; + +function assertDbInvariant(b: boolean): asserts b { + if (!b) { + throw Error("internal invariant failed"); + } +} + +const SqliteError = { + constraintPrimarykey: "SQLITE_CONSTRAINT_PRIMARYKEY", +} as const; + +export type SqliteRowid = number | bigint; + +enum TransactionLevel { + None = 0, + Read = 1, + Write = 2, + VersionChange = 3, +} + +interface ConnectionInfo { + // Database that the connection has + // connected to. + databaseName: string; +} + +interface TransactionInfo { + connectionCookie: string; +} + +interface ScopeIndexInfo { + indexId: SqliteRowid; + keyPath: IDBKeyPath | IDBKeyPath[]; + multiEntry: boolean; + unique: boolean; +} + +interface ScopeInfo { + /** + * Internal ID of the object store. + * Used for fast retrieval, since it's the + * primary key / rowid of the sqlite table. + */ + objectStoreId: SqliteRowid; + + indexMap: Map<string, ScopeIndexInfo>; +} + +interface IndexIterPos { + objectPos: Uint8Array; + indexPos: Uint8Array; +} + +export function serializeKeyPath( + keyPath: string | string[] | null, +): string | null { + if (Array.isArray(keyPath)) { + return "," + keyPath.join(","); + } + return keyPath; +} + +export function deserializeKeyPath( + dbKeyPath: string | null, +): string | string[] | null { + if (dbKeyPath == null) { + return null; + } + if (dbKeyPath[0] === ",") { + const elems = dbKeyPath.split(","); + elems.splice(0, 1); + return elems; + } else { + return dbKeyPath; + } +} + +interface Boundary { + key: Uint8Array; + inclusive: boolean; +} + +function getRangeEndBoundary( + forward: boolean, + range: IDBKeyRange | undefined | null, +): Boundary | undefined { + let endRangeKey: Uint8Array | undefined = undefined; + let endRangeInclusive: boolean = false; + if (range) { + if (forward && range.upper != null) { + endRangeKey = serializeKey(range.upper); + endRangeInclusive = !range.upperOpen; + } else if (!forward && range.lower != null) { + endRangeKey = serializeKey(range.lower); + endRangeInclusive = !range.lowerOpen; + } + } + if (endRangeKey) { + return { + inclusive: endRangeInclusive, + key: endRangeKey, + }; + } + return undefined; +} + +function isOutsideBoundary( + forward: boolean, + endRange: Boundary, + currentKey: Uint8Array, +): boolean { + const cmp = compareSerializedKeys(currentKey, endRange.key); + if (forward && endRange.inclusive && cmp > 0) { + return true; + } else if (forward && !endRange.inclusive && cmp >= 0) { + return true; + } else if (!forward && endRange.inclusive && cmp < 0) { + return true; + } else if (!forward && !endRange.inclusive && cmp <= 0) { + return true; + } + return false; +} + +function compareSerializedKeys(k1: Uint8Array, k2: Uint8Array): number { + // FIXME: Simplify! + let i = 0; + while (1) { + let x1 = i >= k1.length ? -1 : k1[i]; + let x2 = i >= k2.length ? -1 : k2[i]; + if (x1 < x2) { + return -1; + } + if (x1 > x2) { + return 1; + } + if (x1 < 0 && x2 < 0) { + return 0; + } + i++; + } + throw Error("not reached"); +} + +export function expectDbNumber( + resultRow: unknown, + name: string, +): number | bigint { + assertDbInvariant(typeof resultRow === "object" && resultRow != null); + const res = (resultRow as any)[name]; + if (typeof res !== "number") { + throw Error("unexpected type from database"); + } + return res; +} + +export function expectDbString(resultRow: unknown, name: string): string { + assertDbInvariant(typeof resultRow === "object" && resultRow != null); + const res = (resultRow as any)[name]; + if (typeof res !== "string") { + throw Error("unexpected type from database"); + } + return res; +} + +export function expectDbStringOrNull( + resultRow: unknown, + name: string, +): string | null { + assertDbInvariant(typeof resultRow === "object" && resultRow != null); + const res = (resultRow as any)[name]; + if (res == null) { + return null; + } + if (typeof res !== "string") { + throw Error("unexpected type from database"); + } + return res; +} + +export class SqliteBackend implements Backend { + private connectionIdCounter = 1; + private transactionIdCounter = 1; + + trackStats = false; + + accessStats: AccessStats = { + primitiveStatements: 0, // Counted by the sqlite impl + readTransactions: 0, + writeTransactions: 0, + readsPerStore: {}, + readsPerIndex: {}, + readItemsPerIndex: {}, + readItemsPerStore: {}, + writesPerStore: {}, + }; + + /** + * Condition that is triggered whenever a transaction finishes. + */ + private transactionDoneCond: AsyncCondition = new AsyncCondition(); + + /** + * Is the connection blocked because either an open request + * or delete request is being processed? + */ + private connectionBlocked: boolean = false; + + private txLevel: TransactionLevel = TransactionLevel.None; + + private txScope: Map<string, ScopeInfo> = new Map(); + + private connectionMap: Map<string, ConnectionInfo> = new Map(); + + private transactionMap: Map<string, TransactionInfo> = new Map(); + + private sqlPrepCache: Map<string, Sqlite3Statement> = new Map(); + + enableTracing: boolean = true; + + constructor( + public sqliteImpl: Sqlite3Interface, + public db: Sqlite3Database, + ) {} + + private _prep(sql: string): Sqlite3Statement { + const stmt = this.sqlPrepCache.get(sql); + if (stmt) { + return stmt; + } + const newStmt = this.db.prepare(sql); + this.sqlPrepCache.set(sql, newStmt); + return newStmt; + } + + async getIndexRecords( + btx: DatabaseTransaction, + req: IndexGetQuery, + ): Promise<RecordGetResponse> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.Read) { + throw Error("only allowed in read transaction"); + } + const scopeInfo = this.txScope.get(req.objectStoreName); + if (!scopeInfo) { + throw Error("object store not in scope"); + } + const indexInfo = scopeInfo.indexMap.get(req.indexName); + if (!indexInfo) { + throw Error("index not found"); + } + if (req.advancePrimaryKey != null) { + if (req.advanceIndexKey == null) { + throw Error( + "invalid request (advancePrimaryKey without advanceIndexKey)", + ); + } + } + + if (this.enableTracing) { + console.log( + `querying index os=${req.objectStoreName}, idx=${req.indexName}, direction=${req.direction}`, + ); + } + + const forward: boolean = + req.direction === "next" || req.direction === "nextunique"; + + const queryUnique = + req.direction === "nextunique" || req.direction === "prevunique"; + + const indexId = indexInfo.indexId; + const indexUnique = indexInfo.unique; + + let numResults = 0; + const encPrimaryKeys: Uint8Array[] = []; + const encIndexKeys: Uint8Array[] = []; + const indexKeys: IDBValidKey[] = []; + const primaryKeys: IDBValidKey[] = []; + const values: unknown[] = []; + + const endRange = getRangeEndBoundary(forward, req.range); + + const backendThis = this; + + function packResult() { + if (req.resultLevel > ResultLevel.OnlyCount) { + for (let i = 0; i < encPrimaryKeys.length; i++) { + primaryKeys.push(deserializeKey(encPrimaryKeys[i])); + } + for (let i = 0; i < encIndexKeys.length; i++) { + indexKeys.push(deserializeKey(encIndexKeys[i])); + } + if (req.resultLevel === ResultLevel.Full) { + for (let i = 0; i < encPrimaryKeys.length; i++) { + const val = backendThis._getObjectValue( + scopeInfo!.objectStoreId, + encPrimaryKeys[i], + ); + if (!val) { + throw Error("invariant failed: value not found"); + } + values.push(structuredRevive(JSON.parse(val))); + } + } + } + + if (backendThis.enableTracing) { + console.log(`index query returned ${numResults} results`); + console.log(`result prim keys:`, primaryKeys); + console.log(`result index keys:`, indexKeys); + } + + if (backendThis.trackStats) { + const k = `${req.objectStoreName}.${req.indexName}`; + backendThis.accessStats.readsPerIndex[k] = + (backendThis.accessStats.readsPerIndex[k] ?? 0) + 1; + backendThis.accessStats.readItemsPerIndex[k] = + (backendThis.accessStats.readItemsPerIndex[k] ?? 0) + numResults; + } + + return { + count: numResults, + indexKeys: indexKeys, + primaryKeys: + req.resultLevel >= ResultLevel.OnlyKeys ? primaryKeys : undefined, + values: req.resultLevel >= ResultLevel.Full ? values : undefined, + }; + } + + let currentPos = this._startIndex({ + indexId, + indexUnique, + queryUnique, + forward, + }); + + if (!currentPos) { + return packResult(); + } + + if (this.enableTracing && currentPos) { + console.log(`starting iteration at:`); + console.log(`indexKey:`, deserializeKey(currentPos.indexPos)); + console.log(`objectKey:`, deserializeKey(currentPos.objectPos)); + } + + if (req.advanceIndexKey) { + const advanceIndexKey = serializeKey(req.advanceIndexKey); + const advancePrimaryKey = req.advancePrimaryKey + ? serializeKey(req.advancePrimaryKey) + : undefined; + currentPos = this._continueIndex({ + indexId, + indexUnique, + queryUnique, + inclusive: true, + currentPos, + forward, + targetIndexKey: advanceIndexKey, + targetObjectKey: advancePrimaryKey, + }); + if (!currentPos) { + return packResult(); + } + } + + if (req.lastIndexPosition) { + if (this.enableTracing) { + console.log("index query: seeking past last index position"); + console.log("lastObjectPosition", req.lastObjectStorePosition); + console.log("lastIndexPosition", req.lastIndexPosition); + } + const lastIndexPosition = serializeKey(req.lastIndexPosition); + const lastObjectPosition = req.lastObjectStorePosition + ? serializeKey(req.lastObjectStorePosition) + : undefined; + currentPos = this._continueIndex({ + indexId, + indexUnique, + queryUnique, + inclusive: false, + currentPos, + forward, + targetIndexKey: lastIndexPosition, + targetObjectKey: lastObjectPosition, + }); + if (!currentPos) { + return packResult(); + } + } + + if (this.enableTracing && currentPos) { + console.log( + "before range, current index pos", + deserializeKey(currentPos.indexPos), + ); + console.log( + "... current object pos", + deserializeKey(currentPos.objectPos), + ); + } + + if (req.range != null) { + const targetKeyObj = forward ? req.range.lower : req.range.upper; + if (targetKeyObj != null) { + const targetKey = serializeKey(targetKeyObj); + const inclusive = forward ? !req.range.lowerOpen : !req.range.upperOpen; + currentPos = this._continueIndex({ + indexId, + indexUnique, + queryUnique, + inclusive, + currentPos, + forward, + targetIndexKey: targetKey, + }); + } + if (!currentPos) { + return packResult(); + } + } + + if (this.enableTracing && currentPos) { + console.log( + "after range, current pos", + deserializeKey(currentPos.indexPos), + ); + console.log( + "after range, current obj pos", + deserializeKey(currentPos.objectPos), + ); + } + + while (1) { + if (req.limit != 0 && numResults == req.limit) { + break; + } + if (currentPos == null) { + break; + } + if ( + endRange && + isOutsideBoundary(forward, endRange, currentPos.indexPos) + ) { + break; + } + + numResults++; + + if (req.resultLevel > ResultLevel.OnlyCount) { + encPrimaryKeys.push(currentPos.objectPos); + encIndexKeys.push(currentPos.indexPos); + } + + currentPos = backendThis._continueIndex({ + indexId, + indexUnique, + forward, + inclusive: false, + currentPos: undefined, + queryUnique, + targetIndexKey: currentPos.indexPos, + targetObjectKey: currentPos.objectPos, + }); + } + + return packResult(); + } + + // Continue past targetIndexKey (and optionally targetObjectKey) + // in the direction specified by "forward". + // Do nothing if the current position is already past the + // target position. + _continueIndex(req: { + indexId: SqliteRowid; + indexUnique: boolean; + queryUnique: boolean; + forward: boolean; + inclusive: boolean; + currentPos: IndexIterPos | null | undefined; + targetIndexKey: Uint8Array; + targetObjectKey?: Uint8Array; + }): IndexIterPos | undefined { + const currentPos = req.currentPos; + const forward = req.forward; + const dir = forward ? 1 : -1; + if (currentPos) { + // Check that the target position after the current position. + // If not, we just stay at the current position. + const indexCmp = compareSerializedKeys( + currentPos.indexPos, + req.targetIndexKey, + ); + if (dir * indexCmp > 0) { + return currentPos; + } + if (indexCmp === 0) { + if (req.targetObjectKey != null) { + const objectCmp = compareSerializedKeys( + currentPos.objectPos, + req.targetObjectKey, + ); + if (req.inclusive && objectCmp === 0) { + return currentPos; + } + if (dir * objectCmp > 0) { + return currentPos; + } + } else if (req.inclusive) { + return currentPos; + } + } + } + + let stmt: Sqlite3Statement; + + if (req.indexUnique) { + if (req.forward) { + if (req.inclusive) { + stmt = this._prep(sqlUniqueIndexDataContinueForwardInclusive); + } else { + stmt = this._prep(sqlUniqueIndexDataContinueForwardStrict); + } + } else { + if (req.inclusive) { + stmt = this._prep(sqlUniqueIndexDataContinueBackwardInclusive); + } else { + stmt = this._prep(sqlUniqueIndexDataContinueBackwardStrict); + } + } + } else { + if (req.forward) { + if (req.queryUnique || req.targetObjectKey == null) { + if (req.inclusive) { + stmt = this._prep(sqlIndexDataContinueForwardInclusiveUnique); + } else { + stmt = this._prep(sqlIndexDataContinueForwardStrictUnique); + } + } else { + if (req.inclusive) { + stmt = this._prep(sqlIndexDataContinueForwardInclusive); + } else { + stmt = this._prep(sqlIndexDataContinueForwardStrict); + } + } + } else { + if (req.queryUnique || req.targetObjectKey == null) { + if (req.inclusive) { + stmt = this._prep(sqlIndexDataContinueBackwardInclusiveUnique); + } else { + stmt = this._prep(sqlIndexDataContinueBackwardStrictUnique); + } + } else { + if (req.inclusive) { + stmt = this._prep(sqlIndexDataContinueBackwardInclusive); + } else { + stmt = this._prep(sqlIndexDataContinueBackwardStrict); + } + } + } + } + + const res = stmt.getFirst({ + index_id: req.indexId, + index_key: req.targetIndexKey, + object_key: req.targetObjectKey, + }); + + if (res == null) { + return undefined; + } + + assertDbInvariant(typeof res === "object"); + assertDbInvariant("index_key" in res); + const indexKey = res.index_key; + if (indexKey == null) { + return undefined; + } + assertDbInvariant(indexKey instanceof Uint8Array); + assertDbInvariant("object_key" in res); + const objectKey = res.object_key; + if (objectKey == null) { + return undefined; + } + assertDbInvariant(objectKey instanceof Uint8Array); + + return { + indexPos: indexKey, + objectPos: objectKey, + }; + } + + _startIndex(req: { + indexId: SqliteRowid; + indexUnique: boolean; + queryUnique: boolean; + forward: boolean; + }): IndexIterPos | undefined { + let stmt: Sqlite3Statement; + if (req.indexUnique) { + if (req.forward) { + stmt = this._prep(sqlUniqueIndexDataStartForward); + } else { + stmt = this._prep(sqlUniqueIndexDataStartBackward); + } + } else { + if (req.forward) { + stmt = this._prep(sqlIndexDataStartForward); + } else { + if (req.queryUnique) { + stmt = this._prep(sqlIndexDataStartBackwardUnique); + } else { + stmt = this._prep(sqlIndexDataStartBackward); + } + } + } + + const res = stmt.getFirst({ + index_id: req.indexId, + }); + + if (res == null) { + return undefined; + } + + assertDbInvariant(typeof res === "object"); + assertDbInvariant("index_key" in res); + const indexKey = res.index_key; + assertDbInvariant(indexKey instanceof Uint8Array); + assertDbInvariant("object_key" in res); + const objectKey = res.object_key; + assertDbInvariant(objectKey instanceof Uint8Array); + + return { + indexPos: indexKey, + objectPos: objectKey, + }; + } + + async getObjectStoreRecords( + btx: DatabaseTransaction, + req: ObjectStoreGetQuery, + ): Promise<RecordGetResponse> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.Read) { + throw Error("only allowed in read transaction"); + } + const scopeInfo = this.txScope.get(req.objectStoreName); + if (!scopeInfo) { + throw Error( + `object store ${JSON.stringify( + req.objectStoreName, + )} not in transaction scope`, + ); + } + + const forward: boolean = + req.direction === "next" || req.direction === "nextunique"; + + let currentKey = this._startObjectKey(scopeInfo.objectStoreId, forward); + + if (req.advancePrimaryKey != null) { + const targetKey = serializeKey(req.advancePrimaryKey); + currentKey = this._continueObjectKey({ + objectStoreId: scopeInfo.objectStoreId, + forward, + inclusive: true, + currentKey, + targetKey, + }); + } + + if (req.lastObjectStorePosition != null) { + const targetKey = serializeKey(req.lastObjectStorePosition); + currentKey = this._continueObjectKey({ + objectStoreId: scopeInfo.objectStoreId, + forward, + inclusive: false, + currentKey, + targetKey, + }); + } + + if (req.range != null) { + const targetKeyObj = forward ? req.range.lower : req.range.upper; + if (targetKeyObj != null) { + const targetKey = serializeKey(targetKeyObj); + const inclusive = forward ? !req.range.lowerOpen : !req.range.upperOpen; + currentKey = this._continueObjectKey({ + objectStoreId: scopeInfo.objectStoreId, + forward, + inclusive, + currentKey, + targetKey, + }); + } + } + + const endRange = getRangeEndBoundary(forward, req.range); + + let numResults = 0; + const encPrimaryKeys: Uint8Array[] = []; + const primaryKeys: IDBValidKey[] = []; + const values: unknown[] = []; + + while (1) { + if (req.limit != 0 && numResults == req.limit) { + break; + } + if (currentKey == null) { + break; + } + if (endRange && isOutsideBoundary(forward, endRange, currentKey)) { + break; + } + + numResults++; + + if (req.resultLevel > ResultLevel.OnlyCount) { + encPrimaryKeys.push(currentKey); + } + + currentKey = this._continueObjectKey({ + objectStoreId: scopeInfo.objectStoreId, + forward, + inclusive: false, + currentKey: null, + targetKey: currentKey, + }); + } + + if (req.resultLevel > ResultLevel.OnlyCount) { + for (let i = 0; i < encPrimaryKeys.length; i++) { + primaryKeys.push(deserializeKey(encPrimaryKeys[i])); + } + if (req.resultLevel === ResultLevel.Full) { + for (let i = 0; i < encPrimaryKeys.length; i++) { + const val = this._getObjectValue( + scopeInfo.objectStoreId, + encPrimaryKeys[i], + ); + if (!val) { + throw Error("invariant failed: value not found"); + } + values.push(structuredRevive(JSON.parse(val))); + } + } + } + + if (this.trackStats) { + const k = `${req.objectStoreName}`; + this.accessStats.readsPerStore[k] = + (this.accessStats.readsPerStore[k] ?? 0) + 1; + this.accessStats.readItemsPerStore[k] = + (this.accessStats.readItemsPerStore[k] ?? 0) + numResults; + } + + return { + count: numResults, + indexKeys: undefined, + primaryKeys: + req.resultLevel >= ResultLevel.OnlyKeys ? primaryKeys : undefined, + values: req.resultLevel >= ResultLevel.Full ? values : undefined, + }; + } + + _startObjectKey( + objectStoreId: number | bigint, + forward: boolean, + ): Uint8Array | null { + let stmt: Sqlite3Statement; + if (forward) { + stmt = this._prep(sqlObjectDataStartForward); + } else { + stmt = this._prep(sqlObjectDataStartBackward); + } + const res = stmt.getFirst({ + object_store_id: objectStoreId, + }); + if (!res) { + return null; + } + assertDbInvariant(typeof res === "object"); + assertDbInvariant("rkey" in res); + const rkey = res.rkey; + if (!rkey) { + return null; + } + assertDbInvariant(rkey instanceof Uint8Array); + return rkey; + } + + // Result *must* be past targetKey in the direction + // specified by "forward". + _continueObjectKey(req: { + objectStoreId: number | bigint; + forward: boolean; + currentKey: Uint8Array | null; + targetKey: Uint8Array; + inclusive: boolean; + }): Uint8Array | null { + const { forward, currentKey, targetKey } = req; + const dir = forward ? 1 : -1; + if (currentKey) { + const objCmp = compareSerializedKeys(currentKey, targetKey); + if (objCmp === 0 && req.inclusive) { + return currentKey; + } + if (dir * objCmp > 0) { + return currentKey; + } + } + + let stmt: Sqlite3Statement; + + if (req.inclusive) { + if (req.forward) { + stmt = this._prep(sqlObjectDataContinueForwardInclusive); + } else { + stmt = this._prep(sqlObjectDataContinueBackwardInclusive); + } + } else { + if (req.forward) { + stmt = this._prep(sqlObjectDataContinueForward); + } else { + stmt = this._prep(sqlObjectDataContinueBackward); + } + } + + const res = stmt.getFirst({ + object_store_id: req.objectStoreId, + x: req.targetKey, + }); + + if (!res) { + return null; + } + + assertDbInvariant(typeof res === "object"); + assertDbInvariant("rkey" in res); + const rkey = res.rkey; + if (!rkey) { + return null; + } + assertDbInvariant(rkey instanceof Uint8Array); + return rkey; + } + + _getObjectValue( + objectStoreId: number | bigint, + key: Uint8Array, + ): string | undefined { + const stmt = this._prep(sqlObjectDataValueFromKey); + const res = stmt.getFirst({ + object_store_id: objectStoreId, + key: key, + }); + if (!res) { + return undefined; + } + assertDbInvariant(typeof res === "object"); + assertDbInvariant("value" in res); + assertDbInvariant(typeof res.value === "string"); + return res.value; + } + + getObjectStoreMeta( + dbConn: DatabaseConnection, + objectStoreName: string, + ): ObjectStoreMeta | undefined { + // FIXME: Use cached info from the connection for this! + const connInfo = this.connectionMap.get(dbConn.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + const objRes = this._prep(sqlGetObjectStoreMetaByName).getFirst({ + name: objectStoreName, + database_name: connInfo.databaseName, + }); + if (!objRes) { + throw Error("object store not found"); + } + const objectStoreId = expectDbNumber(objRes, "id"); + const keyPath = deserializeKeyPath( + expectDbStringOrNull(objRes, "key_path"), + ); + const autoInc = expectDbNumber(objRes, "auto_increment"); + const indexSet: string[] = []; + const indexRes = this._prep(sqlGetIndexesByObjectStoreId).getAll({ + object_store_id: objectStoreId, + }); + for (const idxInfo of indexRes) { + const indexName = expectDbString(idxInfo, "name"); + indexSet.push(indexName); + } + return { + keyPath, + autoIncrement: autoInc != 0, + indexSet, + }; + } + + getIndexMeta( + dbConn: DatabaseConnection, + objectStoreName: string, + indexName: string, + ): IndexMeta | undefined { + // FIXME: Use cached info from the connection for this! + const connInfo = this.connectionMap.get(dbConn.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + const objRes = this._prep(sqlGetObjectStoreMetaByName).getFirst({ + name: objectStoreName, + database_name: connInfo.databaseName, + }); + if (!objRes) { + throw Error("object store not found"); + } + const objectStoreId = expectDbNumber(objRes, "id"); + const idxInfo = this._prep(sqlGetIndexByName).getFirst({ + object_store_id: objectStoreId, + name: indexName, + }); + if (!idxInfo) { + throw Error( + `index ${indexName} on object store ${objectStoreName} not found`, + ); + } + const indexUnique = expectDbNumber(idxInfo, "unique_index"); + const indexMultiEntry = expectDbNumber(idxInfo, "multientry"); + const indexKeyPath = deserializeKeyPath( + expectDbString(idxInfo, "key_path"), + ); + if (!indexKeyPath) { + throw Error("db inconsistent"); + } + return { + keyPath: indexKeyPath, + multiEntry: indexMultiEntry != 0, + unique: indexUnique != 0, + }; + } + + async getDatabases(): Promise<BridgeIDBDatabaseInfo[]> { + const dbList = this._prep(sqlListDatabases).getAll(); + let res: BridgeIDBDatabaseInfo[] = []; + for (const r of dbList) { + res.push({ + name: (r as any).name, + version: (r as any).version, + }); + } + + return res; + } + + private _loadObjectStoreNames(databaseName: string): string[] { + const objectStoreNames: string[] = []; + const storesRes = this._prep(sqlGetObjectStoresByDatabase).getAll({ + database_name: databaseName, + }); + for (const res of storesRes) { + assertDbInvariant(res != null && typeof res === "object"); + assertDbInvariant("name" in res); + const storeName = res.name; + assertDbInvariant(typeof storeName === "string"); + objectStoreNames.push(storeName); + } + return objectStoreNames; + } + + async connectDatabase(databaseName: string): Promise<ConnectResult> { + const connectionId = this.connectionIdCounter++; + const connectionCookie = `connection-${connectionId}`; + + // Wait until no transaction is active anymore. + while (1) { + if (this.txLevel == TransactionLevel.None) { + break; + } + await this.transactionDoneCond.wait(); + } + + this._prep(sqlBegin).run(); + const versionRes = this._prep(sqlGetDatabaseVersion).getFirst({ + name: databaseName, + }); + let ver: number; + if (versionRes == undefined) { + this._prep(sqlCreateDatabase).run({ name: databaseName }); + ver = 0; + } else { + const verNum = expectDbNumber(versionRes, "version"); + assertDbInvariant(typeof verNum === "number"); + ver = verNum; + } + const objectStoreNames: string[] = this._loadObjectStoreNames(databaseName); + + this._prep(sqlCommit).run(); + + this.connectionMap.set(connectionCookie, { + databaseName: databaseName, + }); + + return { + conn: { + connectionCookie, + }, + version: ver, + objectStores: objectStoreNames, + }; + } + + private _loadScopeInfo(connInfo: ConnectionInfo, storeName: string): void { + const objRes = this._prep(sqlGetObjectStoreMetaByName).getFirst({ + name: storeName, + database_name: connInfo.databaseName, + }); + if (!objRes) { + throw Error("object store not found"); + } + const objectStoreId = expectDbNumber(objRes, "id"); + const indexRes = this._prep(sqlGetIndexesByObjectStoreId).getAll({ + object_store_id: objectStoreId, + }); + if (!indexRes) { + throw Error("db inconsistent"); + } + const indexMap = new Map<string, ScopeIndexInfo>(); + for (const idxInfo of indexRes) { + const indexId = expectDbNumber(idxInfo, "id"); + const indexName = expectDbString(idxInfo, "name"); + const indexUnique = expectDbNumber(idxInfo, "unique_index"); + const indexMultiEntry = expectDbNumber(idxInfo, "multientry"); + const indexKeyPath = deserializeKeyPath( + expectDbString(idxInfo, "key_path"), + ); + if (!indexKeyPath) { + throw Error("db inconsistent"); + } + indexMap.set(indexName, { + indexId, + keyPath: indexKeyPath, + multiEntry: indexMultiEntry != 0, + unique: indexUnique != 0, + }); + } + this.txScope.set(storeName, { + objectStoreId, + indexMap, + }); + } + + async beginTransaction( + conn: DatabaseConnection, + objectStores: string[], + mode: IDBTransactionMode, + ): Promise<DatabaseTransaction> { + const connInfo = this.connectionMap.get(conn.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + const transactionCookie = `tx-${this.transactionIdCounter++}`; + + while (1) { + if (this.txLevel === TransactionLevel.None) { + break; + } + await this.transactionDoneCond.wait(); + } + + if (this.trackStats) { + if (mode === "readonly") { + this.accessStats.readTransactions++; + } else if (mode === "readwrite") { + this.accessStats.writeTransactions++; + } + } + + this._prep(sqlBegin).run(); + if (mode === "readonly") { + this.txLevel = TransactionLevel.Read; + } else if (mode === "readwrite") { + this.txLevel = TransactionLevel.Write; + } + + this.transactionMap.set(transactionCookie, { + connectionCookie: conn.connectionCookie, + }); + + // FIXME: We should check this + // if (this.txScope.size != 0) { + // // Something didn't clean up! + // throw Error("scope not empty"); + // } + this.txScope.clear(); + + // FIXME: Use cached info from connection? + for (const storeName of objectStores) { + this._loadScopeInfo(connInfo, storeName); + } + + return { + transactionCookie, + }; + } + + async enterVersionChange( + conn: DatabaseConnection, + newVersion: number, + ): Promise<DatabaseTransaction> { + const connInfo = this.connectionMap.get(conn.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.enableTracing) { + console.log( + `entering version change transaction (conn ${conn.connectionCookie}), newVersion=${newVersion}`, + ); + } + const transactionCookie = `tx-vc-${this.transactionIdCounter++}`; + + while (1) { + if (this.txLevel === TransactionLevel.None) { + break; + } + await this.transactionDoneCond.wait(); + } + + // FIXME: We should check this + // if (this.txScope.size != 0) { + // // Something didn't clean up! + // throw Error("scope not empty"); + // } + this.txScope.clear(); + + if (this.enableTracing) { + console.log(`version change transaction unblocked`); + } + + this._prep(sqlBegin).run(); + this.txLevel = TransactionLevel.VersionChange; + + this.transactionMap.set(transactionCookie, { + connectionCookie: conn.connectionCookie, + }); + + this._prep(sqlUpdateDbVersion).run({ + name: connInfo.databaseName, + version: newVersion, + }); + + const objectStoreNames = this._loadObjectStoreNames(connInfo.databaseName); + + // FIXME: Use cached info from connection? + for (const storeName of objectStoreNames) { + this._loadScopeInfo(connInfo, storeName); + } + + return { + transactionCookie, + }; + } + + async deleteDatabase(databaseName: string): Promise<void> { + // FIXME: Wait until connection queue is not blocked + // FIXME: To properly implement the spec semantics, maybe + // split delete into prepareDelete and executeDelete? + + while (this.txLevel !== TransactionLevel.None) { + await this.transactionDoneCond.wait(); + } + + this._prep(sqlBegin).run(); + const objectStoreNames = this._loadObjectStoreNames(databaseName); + for (const storeName of objectStoreNames) { + const objRes = this._prep(sqlGetObjectStoreMetaByName).getFirst({ + name: storeName, + database_name: databaseName, + }); + if (!objRes) { + throw Error("object store not found"); + } + const objectStoreId = expectDbNumber(objRes, "id"); + const indexRes = this._prep(sqlGetIndexesByObjectStoreId).getAll({ + object_store_id: objectStoreId, + }); + if (!indexRes) { + throw Error("db inconsistent"); + } + const indexMap = new Map<string, ScopeIndexInfo>(); + for (const idxInfo of indexRes) { + const indexId = expectDbNumber(idxInfo, "id"); + const indexName = expectDbString(idxInfo, "name"); + const indexUnique = expectDbNumber(idxInfo, "unique_index"); + const indexMultiEntry = expectDbNumber(idxInfo, "multientry"); + const indexKeyPath = deserializeKeyPath( + expectDbString(idxInfo, "key_path"), + ); + if (!indexKeyPath) { + throw Error("db inconsistent"); + } + indexMap.set(indexName, { + indexId, + keyPath: indexKeyPath, + multiEntry: indexMultiEntry != 0, + unique: indexUnique != 0, + }); + } + this.txScope.set(storeName, { + objectStoreId, + indexMap, + }); + + for (const indexInfo of indexMap.values()) { + let stmt: Sqlite3Statement; + if (indexInfo.unique) { + stmt = this._prep(sqlIUniqueIndexDataDeleteAll); + } else { + stmt = this._prep(sqlIndexDataDeleteAll); + } + stmt.run({ + index_id: indexInfo.indexId, + }); + this._prep(sqlIndexDelete).run({ + index_id: indexInfo.indexId, + }); + } + this._prep(sqlObjectDataDeleteAll).run({ + object_store_id: objectStoreId, + }); + this._prep(sqlObjectStoreDelete).run({ + object_store_id: objectStoreId, + }); + } + this._prep(sqlDeleteDatabase).run({ + name: databaseName, + }); + this._prep(sqlCommit).run(); + } + + async close(db: DatabaseConnection): Promise<void> { + const connInfo = this.connectionMap.get(db.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + // FIXME: What if we're in a transaction? Does the backend interface allow this? + // if (this.txLevel !== TransactionLevel.None) { + // throw Error("can't close while in transaction"); + // } + if (this.enableTracing) { + console.log(`closing connection ${db.connectionCookie}`); + } + this.connectionMap.delete(db.connectionCookie); + } + + renameObjectStore( + btx: DatabaseTransaction, + oldName: string, + newName: string, + ): void { + if (this.enableTracing) { + console.log(`renaming object store '${oldName}' to '${newName}'`); + } + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction required"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("not connected"); + } + // FIXME: Would be much nicer with numeric UID handles + const scopeInfo = this.txScope.get(oldName); + if (!scopeInfo) { + throw Error("object store not found"); + } + this.txScope.delete(oldName); + this.txScope.set(newName, scopeInfo); + this._prep(sqlRenameObjectStore).run({ + object_store_id: scopeInfo.objectStoreId, + name: newName, + }); + } + + renameIndex( + btx: DatabaseTransaction, + objectStoreName: string, + oldIndexName: string, + newIndexName: string, + ): void { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction required"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("not connected"); + } + // FIXME: Would be much nicer with numeric UID handles + const scopeInfo = this.txScope.get(objectStoreName); + if (!scopeInfo) { + throw Error("object store not found"); + } + const indexInfo = scopeInfo.indexMap.get(oldIndexName); + if (!indexInfo) { + throw Error("index not found"); + } + // FIXME: Would also be much nicer with numeric UID handles + scopeInfo.indexMap.delete(oldIndexName); + scopeInfo.indexMap.set(newIndexName, indexInfo); + this._prep(sqlRenameIndex).run({ + index_id: indexInfo.indexId, + name: newIndexName, + }); + } + + deleteObjectStore(btx: DatabaseTransaction, name: string): void { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction required"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("not connected"); + } + // FIXME: Would be much nicer with numeric UID handles + const scopeInfo = this.txScope.get(name); + if (!scopeInfo) { + throw Error("object store not found"); + } + for (const indexInfo of scopeInfo.indexMap.values()) { + let stmt: Sqlite3Statement; + if (indexInfo.unique) { + stmt = this._prep(sqlIUniqueIndexDataDeleteAll); + } else { + stmt = this._prep(sqlIndexDataDeleteAll); + } + stmt.run({ + index_id: indexInfo.indexId, + }); + this._prep(sqlIndexDelete).run({ + index_id: indexInfo.indexId, + }); + } + this._prep(sqlObjectDataDeleteAll).run({ + object_store_id: scopeInfo.objectStoreId, + }); + this._prep(sqlObjectStoreDelete).run({ + object_store_id: scopeInfo.objectStoreId, + }); + this.txScope.delete(name); + } + + deleteIndex( + btx: DatabaseTransaction, + objectStoreName: string, + indexName: string, + ): void { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction required"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("not connected"); + } + // FIXME: Would be much nicer with numeric UID handles + const scopeInfo = this.txScope.get(objectStoreName); + if (!scopeInfo) { + throw Error("object store not found"); + } + const indexInfo = scopeInfo.indexMap.get(indexName); + if (!indexInfo) { + throw Error("index not found"); + } + scopeInfo.indexMap.delete(indexName); + let stmt: Sqlite3Statement; + if (indexInfo.unique) { + stmt = this._prep(sqlIUniqueIndexDataDeleteAll); + } else { + stmt = this._prep(sqlIndexDataDeleteAll); + } + stmt.run({ + index_id: indexInfo.indexId, + }); + this._prep(sqlIndexDelete).run({ + index_id: indexInfo.indexId, + }); + } + + async rollback(btx: DatabaseTransaction): Promise<void> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + if (this.enableTracing) { + console.log(`rolling back transaction ${btx.transactionCookie}`); + } + if (this.txLevel === TransactionLevel.None) { + return; + } + this._prep(sqlRollback).run(); + this.txLevel = TransactionLevel.None; + this.transactionMap.delete(btx.transactionCookie); + this.txScope.clear(); + this.transactionDoneCond.trigger(); + } + + async commit(btx: DatabaseTransaction): Promise<void> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + if (this.enableTracing) { + console.log(`committing transaction ${btx.transactionCookie}`); + } + if (this.txLevel === TransactionLevel.None) { + return; + } + this._prep(sqlCommit).run(); + this.txLevel = TransactionLevel.None; + this.txScope.clear(); + this.transactionMap.delete(btx.transactionCookie); + this.transactionDoneCond.trigger(); + } + + createObjectStore( + btx: DatabaseTransaction, + name: string, + keyPath: string | string[] | null, + autoIncrement: boolean, + ): void { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.VersionChange) { + throw Error("only allowed in versionchange transaction"); + } + if (this.txScope.has(name)) { + throw Error("object store already exists"); + } + let myKeyPath = serializeKeyPath(keyPath); + const runRes = this._prep(sqlCreateObjectStore).run({ + name, + key_path: myKeyPath, + auto_increment: autoIncrement ? 1 : 0, + database_name: connInfo.databaseName, + }); + this.txScope.set(name, { + objectStoreId: runRes.lastInsertRowid, + indexMap: new Map(), + }); + } + + createIndex( + btx: DatabaseTransaction, + indexName: string, + objectStoreName: string, + keyPath: string | string[], + multiEntry: boolean, + unique: boolean, + ): void { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.VersionChange) { + throw Error("only allowed in versionchange transaction"); + } + const scopeInfo = this.txScope.get(objectStoreName); + if (!scopeInfo) { + throw Error("object store does not exist, can't create index"); + } + if (scopeInfo.indexMap.has(indexName)) { + throw Error("index already exists"); + } + + if (this.enableTracing) { + console.log(`creating index "${indexName}"`); + } + + const res = this._prep(sqlCreateIndex).run({ + object_store_id: scopeInfo.objectStoreId, + name: indexName, + key_path: serializeKeyPath(keyPath), + unique: unique ? 1 : 0, + multientry: multiEntry ? 1 : 0, + }); + const scopeIndexInfo: ScopeIndexInfo = { + indexId: res.lastInsertRowid, + keyPath, + multiEntry, + unique, + }; + scopeInfo.indexMap.set(indexName, scopeIndexInfo); + + // FIXME: We can't use an iterator here, as it's not allowed to + // execute a write statement while the iterator executes. + // Maybe do multiple selects instead of loading everything into memory? + const keyRowsRes = this._prep(sqlObjectDataGetAll).getAll({ + object_store_id: scopeInfo.objectStoreId, + }); + + for (const keyRow of keyRowsRes) { + assertDbInvariant(typeof keyRow === "object" && keyRow != null); + assertDbInvariant("key" in keyRow); + assertDbInvariant("value" in keyRow); + assertDbInvariant(typeof keyRow.value === "string"); + const key = keyRow.key; + const value = structuredRevive(JSON.parse(keyRow.value)); + assertDbInvariant(key instanceof Uint8Array); + try { + this.insertIntoIndex(scopeIndexInfo, key, value); + } catch (e) { + // FIXME: Catch this in insertIntoIndex! + if (e instanceof DataError) { + // https://www.w3.org/TR/IndexedDB-2/#object-store-storage-operation + // Do nothing + } else { + throw e; + } + } + } + } + + async deleteRecord( + btx: DatabaseTransaction, + objectStoreName: string, + range: BridgeIDBKeyRange, + ): Promise<void> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.Write) { + throw Error("store operation only allowed while running a transaction"); + } + const scopeInfo = this.txScope.get(objectStoreName); + if (!scopeInfo) { + throw Error( + `object store ${JSON.stringify( + objectStoreName, + )} not in transaction scope`, + ); + } + + // PERF: We delete keys one-by-one here. + // Instead, we could do it with a single + // delete query for the object data / index data. + + let currKey: Uint8Array | null = null; + + if (range.lower != null) { + const targetKey = serializeKey(range.lower); + currKey = this._continueObjectKey({ + objectStoreId: scopeInfo.objectStoreId, + currentKey: null, + forward: true, + inclusive: true, + targetKey, + }); + } else { + currKey = this._startObjectKey(scopeInfo.objectStoreId, true); + } + + let upperBound: Uint8Array | undefined; + if (range.upper != null) { + upperBound = serializeKey(range.upper); + } + + // loop invariant: (currKey is undefined) or (currKey is a valid key) + while (true) { + if (!currKey) { + break; + } + + // FIXME: Check if we're past the range! + if (upperBound != null) { + const cmp = compareSerializedKeys(currKey, upperBound); + if (cmp > 0) { + break; + } + if (cmp == 0 && range.upperOpen) { + break; + } + } + + // Now delete! + + this._prep(sqlObjectDataDeleteKey).run({ + object_store_id: scopeInfo.objectStoreId, + key: currKey, + }); + + for (const index of scopeInfo.indexMap.values()) { + let stmt: Sqlite3Statement; + if (index.unique) { + stmt = this._prep(sqlUniqueIndexDataDeleteKey); + } else { + stmt = this._prep(sqlIndexDataDeleteKey); + } + stmt.run({ + index_id: index.indexId, + object_key: currKey, + }); + } + + currKey = this._continueObjectKey({ + objectStoreId: scopeInfo.objectStoreId, + currentKey: null, + forward: true, + inclusive: false, + targetKey: currKey, + }); + } + } + + async storeRecord( + btx: DatabaseTransaction, + storeReq: RecordStoreRequest, + ): Promise<RecordStoreResponse> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.Write) { + throw Error("store operation only allowed while running a transaction"); + } + const scopeInfo = this.txScope.get(storeReq.objectStoreName); + if (!scopeInfo) { + throw Error( + `object store ${JSON.stringify( + storeReq.objectStoreName, + )} not in transaction scope`, + ); + } + const metaRes = this._prep(sqlGetObjectStoreMetaById).getFirst({ + id: scopeInfo.objectStoreId, + }); + if (metaRes === undefined) { + throw Error( + `object store ${JSON.stringify( + storeReq.objectStoreName, + )} does not exist`, + ); + } + assertDbInvariant(!!metaRes && typeof metaRes === "object"); + assertDbInvariant("key_path" in metaRes); + assertDbInvariant("auto_increment" in metaRes); + const dbKeyPath = metaRes.key_path; + assertDbInvariant(dbKeyPath === null || typeof dbKeyPath === "string"); + const keyPath = deserializeKeyPath(dbKeyPath); + const autoIncrement = metaRes.auto_increment; + assertDbInvariant(typeof autoIncrement === "number"); + + let key; + let value; + let updatedKeyGenerator: number | undefined; + + if (storeReq.storeLevel === StoreLevel.UpdateExisting) { + if (storeReq.key == null) { + throw Error("invalid update request (key not given)"); + } + key = storeReq.key; + value = storeReq.value; + } else { + if (keyPath != null && storeReq.key !== undefined) { + // If in-line keys are used, a key can't be explicitly specified. + throw new DataError(); + } + + const storeKeyResult = makeStoreKeyValue({ + value: storeReq.value, + key: storeReq.key, + currentKeyGenerator: autoIncrement, + autoIncrement: autoIncrement != 0, + keyPath: keyPath, + }); + + if (autoIncrement != 0) { + updatedKeyGenerator = storeKeyResult.updatedKeyGenerator; + } + + key = storeKeyResult.key; + value = storeKeyResult.value; + } + + const serializedObjectKey = serializeKey(key); + + const existingObj = this._getObjectValue( + scopeInfo.objectStoreId, + serializedObjectKey, + ); + + if (storeReq.storeLevel === StoreLevel.NoOverwrite) { + if (existingObj) { + throw new ConstraintError(); + } + } + + this._prep(sqlInsertObjectData).run({ + object_store_id: scopeInfo.objectStoreId, + key: serializedObjectKey, + value: JSON.stringify(structuredEncapsulate(value)), + }); + + if (autoIncrement != 0) { + this._prep(sqlUpdateAutoIncrement).run({ + object_store_id: scopeInfo.objectStoreId, + auto_increment: updatedKeyGenerator, + }); + } + + for (const [k, indexInfo] of scopeInfo.indexMap.entries()) { + if (existingObj) { + this.deleteFromIndex( + indexInfo.indexId, + indexInfo.unique, + serializedObjectKey, + ); + } + + try { + this.insertIntoIndex(indexInfo, serializedObjectKey, value); + } catch (e) { + // FIXME: handle this in insertIntoIndex! + if (e instanceof DataError) { + // We don't propagate this error here. + continue; + } + throw e; + } + } + + if (this.trackStats) { + this.accessStats.writesPerStore[storeReq.objectStoreName] = + (this.accessStats.writesPerStore[storeReq.objectStoreName] ?? 0) + 1; + } + + return { + key: key, + }; + } + + private deleteFromIndex( + indexId: SqliteRowid, + indexUnique: boolean, + objectKey: Uint8Array, + ): void { + let stmt: Sqlite3Statement; + if (indexUnique) { + stmt = this._prep(sqlUniqueIndexDataDeleteKey); + } else { + stmt = this._prep(sqlIndexDataDeleteKey); + } + stmt.run({ + index_id: indexId, + object_key: objectKey, + }); + } + + private insertIntoIndex( + indexInfo: ScopeIndexInfo, + primaryKey: Uint8Array, + value: any, + ): void { + const indexKeys = getIndexKeys( + value, + indexInfo.keyPath, + indexInfo.multiEntry, + ); + if (!indexKeys.length) { + return; + } + + let stmt; + if (indexInfo.unique) { + stmt = this._prep(sqlInsertUniqueIndexData); + } else { + stmt = this._prep(sqlInsertIndexData); + } + + for (const indexKey of indexKeys) { + // FIXME: Re-throw correct error for unique index violations + const serializedIndexKey = serializeKey(indexKey); + try { + stmt.run({ + index_id: indexInfo.indexId, + object_key: primaryKey, + index_key: serializedIndexKey, + }); + } catch (e: any) { + if (e.code === SqliteError.constraintPrimarykey) { + throw new ConstraintError(); + } + throw e; + } + } + } + + async clearObjectStore( + btx: DatabaseTransaction, + objectStoreName: string, + ): Promise<void> { + const txInfo = this.transactionMap.get(btx.transactionCookie); + if (!txInfo) { + throw Error("transaction not found"); + } + const connInfo = this.connectionMap.get(txInfo.connectionCookie); + if (!connInfo) { + throw Error("connection not found"); + } + if (this.txLevel < TransactionLevel.Write) { + throw Error("store operation only allowed while running a transaction"); + } + const scopeInfo = this.txScope.get(objectStoreName); + if (!scopeInfo) { + throw Error( + `object store ${JSON.stringify( + objectStoreName, + )} not in transaction scope`, + ); + } + + this._prep(sqlClearObjectStore).run({ + object_store_id: scopeInfo.objectStoreId, + }); + + for (const index of scopeInfo.indexMap.values()) { + let stmt: Sqlite3Statement; + if (index.unique) { + stmt = this._prep(sqlClearUniqueIndexData); + } else { + stmt = this._prep(sqlClearIndexData); + } + stmt.run({ + index_id: index.indexId, + }); + } + } +} + +const schemaSql = ` +CREATE TABLE IF NOT EXISTS databases +( name TEXT PRIMARY KEY +, version INTEGER NOT NULL +); + +CREATE TABLE IF NOT EXISTS object_stores +( id INTEGER PRIMARY KEY +, database_name NOT NULL +, name TEXT NOT NULL +, key_path TEXT +, auto_increment INTEGER NOT NULL DEFAULT 0 +, FOREIGN KEY (database_name) + REFERENCES databases(name) +); + +CREATE TABLE IF NOT EXISTS indexes +( id INTEGER PRIMARY KEY +, object_store_id INTEGER NOT NULL +, name TEXT NOT NULL +, key_path TEXT NOT NULL +, unique_index INTEGER NOT NULL +, multientry INTEGER NOT NULL +, FOREIGN KEY (object_store_id) + REFERENCES object_stores(id) +); + +CREATE TABLE IF NOT EXISTS object_data +( object_store_id INTEGER NOT NULL +, key BLOB NOT NULL +, value TEXT NOT NULL +, PRIMARY KEY (object_store_id, key) +); + +CREATE TABLE IF NOT EXISTS index_data +( index_id INTEGER NOT NULL +, index_key BLOB NOT NULL +, object_key BLOB NOT NULL +, PRIMARY KEY (index_id, index_key, object_key) +, FOREIGN KEY (index_id) + REFERENCES indexes(id) +); + +CREATE TABLE IF NOT EXISTS unique_index_data +( index_id INTEGER NOT NULL +, index_key BLOB NOT NULL +, object_key BLOB NOT NULL +, PRIMARY KEY (index_id, index_key) +, FOREIGN KEY (index_id) + REFERENCES indexes(id) +); +`; + +const sqlClearObjectStore = ` +DELETE FROM object_data WHERE object_store_id=$object_store_id`; + +const sqlClearIndexData = ` +DELETE FROM index_data WHERE index_id=$index_id`; + +const sqlClearUniqueIndexData = ` +DELETE FROM unique_index_data WHERE index_id=$index_id`; + +const sqlListDatabases = ` +SELECT name, version FROM databases; +`; + +const sqlGetDatabaseVersion = ` +SELECT version FROM databases WHERE name=$name; +`; + +const sqlBegin = `BEGIN;`; +const sqlCommit = `COMMIT;`; +const sqlRollback = `ROLLBACK;`; + +const sqlCreateDatabase = ` +INSERT INTO databases (name, version) VALUES ($name, 1); +`; + +const sqlDeleteDatabase = ` +DELETE FROM databases +WHERE name=$name; +`; + +const sqlCreateObjectStore = ` +INSERT INTO object_stores (name, database_name, key_path, auto_increment) + VALUES ($name, $database_name, $key_path, $auto_increment); +`; + +const sqlObjectStoreDelete = ` +DELETE FROM object_stores +WHERE id=$object_store_id;`; + +const sqlObjectDataDeleteAll = ` +DELETE FROM object_data +WHERE object_store_id=$object_store_id`; + +const sqlIndexDelete = ` +DELETE FROM indexes +WHERE id=$index_id; +`; + +const sqlIndexDataDeleteAll = ` +DELETE FROM index_data +WHERE index_id=$index_id; +`; + +const sqlIUniqueIndexDataDeleteAll = ` +DELETE FROM unique_index_data +WHERE index_id=$index_id; +`; + +const sqlCreateIndex = ` +INSERT INTO indexes (object_store_id, name, key_path, unique_index, multientry) + VALUES ($object_store_id, $name, $key_path, $unique, $multientry); +`; + +const sqlInsertIndexData = ` +INSERT INTO index_data (index_id, object_key, index_key) + VALUES ($index_id, $object_key, $index_key);`; + +const sqlInsertUniqueIndexData = ` +INSERT INTO unique_index_data (index_id, object_key, index_key) + VALUES ($index_id, $object_key, $index_key);`; + +const sqlUpdateDbVersion = ` +UPDATE databases + SET version=$version + WHERE name=$name; +`; + +const sqlRenameObjectStore = ` +UPDATE object_stores + SET name=$name + WHERE id=$object_store_id`; + +const sqlRenameIndex = ` +UPDATE indexes + SET name=$name + WHERE index_id=$index_id`; + +const sqlGetObjectStoresByDatabase = ` +SELECT id, name, key_path, auto_increment +FROM object_stores +WHERE database_name=$database_name; +`; + +const sqlGetObjectStoreMetaById = ` +SELECT key_path, auto_increment +FROM object_stores +WHERE id = $id; +`; + +const sqlGetObjectStoreMetaByName = ` +SELECT id, key_path, auto_increment +FROM object_stores +WHERE database_name=$database_name AND name=$name; +`; + +const sqlGetIndexesByObjectStoreId = ` +SELECT id, name, key_path, unique_index, multientry +FROM indexes +WHERE object_store_id=$object_store_id +`; + +const sqlGetIndexByName = ` +SELECT id, key_path, unique_index, multientry +FROM indexes +WHERE object_store_id=$object_store_id + AND name=$name +`; + +const sqlInsertObjectData = ` +INSERT OR REPLACE INTO object_data(object_store_id, key, value) + VALUES ($object_store_id, $key, $value); +`; + +const sqlUpdateAutoIncrement = ` +UPDATE object_stores + SET auto_increment=$auto_increment + WHERE id=$object_store_id +`; + +const sqlObjectDataValueFromKey = ` +SELECT value FROM object_data + WHERE object_store_id=$object_store_id + AND key=$key; +`; + +const sqlObjectDataGetAll = ` +SELECT key, value FROM object_data + WHERE object_store_id=$object_store_id;`; + +const sqlObjectDataStartForward = ` +SELECT min(key) as rkey FROM object_data + WHERE object_store_id=$object_store_id;`; + +const sqlObjectDataStartBackward = ` +SELECT max(key) as rkey FROM object_data + WHERE object_store_id=$object_store_id;`; + +const sqlObjectDataContinueForward = ` +SELECT min(key) as rkey FROM object_data + WHERE object_store_id=$object_store_id + AND key > $x;`; + +const sqlObjectDataContinueBackward = ` +SELECT max(key) as rkey FROM object_data + WHERE object_store_id=$object_store_id + AND key < $x;`; + +const sqlObjectDataContinueForwardInclusive = ` +SELECT min(key) as rkey FROM object_data + WHERE object_store_id=$object_store_id + AND key >= $x;`; + +const sqlObjectDataContinueBackwardInclusive = ` +SELECT max(key) as rkey FROM object_data + WHERE object_store_id=$object_store_id + AND key <= $x;`; + +const sqlObjectDataDeleteKey = ` +DELETE FROM object_data + WHERE object_store_id=$object_store_id AND + key=$key`; + +const sqlIndexDataDeleteKey = ` +DELETE FROM index_data + WHERE index_id=$index_id AND + object_key=$object_key; +`; + +const sqlUniqueIndexDataDeleteKey = ` +DELETE FROM unique_index_data + WHERE index_id=$index_id AND + object_key=$object_key; +`; + +// "next" or "nextunique" on a non-unique index +const sqlIndexDataStartForward = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id + ORDER BY index_key, object_key + LIMIT 1; +`; + +// start a "next" or "nextunique" on a unique index +const sqlUniqueIndexDataStartForward = ` +SELECT index_key, object_key FROM unique_index_data + WHERE index_id=$index_id + ORDER BY index_key, object_key + LIMIT 1; +`; + +// start a "prev" or "prevunique" on a unique index +const sqlUniqueIndexDataStartBackward = ` +SELECT index_key, object_key FROM unique_index_data + WHERE index_id=$index_id + ORDER BY index_key DESC, object_key DESC + LIMIT 1 +`; + +// start a "prevunique" query on a non-unique index +const sqlIndexDataStartBackwardUnique = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id + ORDER BY index_key DESC, object_key ASC + LIMIT 1 +`; + +// start a "prev" query on a non-unique index +const sqlIndexDataStartBackward = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id + ORDER BY index_key DESC, object_key DESC + LIMIT 1 +`; + +// continue a "next" query, strictly go to a further key +const sqlIndexDataContinueForwardStrict = ` +SELECT index_key, object_key FROM index_data + WHERE + index_id=$index_id AND + ((index_key = $index_key AND object_key > $object_key) OR + (index_key > $index_key)) + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "next" query, go to at least the specified key +const sqlIndexDataContinueForwardInclusive = ` +SELECT index_key, object_key FROM index_data + WHERE + index_id=$index_id AND + ((index_key = $index_key AND object_key >= $object_key) OR + (index_key > $index_key)) + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "prev" query +const sqlIndexDataContinueBackwardStrict = ` +SELECT index_key, object_key FROM index_data + WHERE + index_id=$index_id AND + ((index_key = $index_key AND object_key < $object_key) OR + (index_key < $index_key)) + ORDER BY index_key DESC, object_key DESC + LIMIT 1; +`; + +// continue a "prev" query +const sqlIndexDataContinueBackwardInclusive = ` +SELECT index_key, object_key FROM index_data + WHERE + index_id=$index_id AND + ((index_key = $index_key AND object_key <= $object_key) OR + (index_key < $index_key)) + ORDER BY index_key DESC, object_key DESC + LIMIT 1; +`; + +// continue a "prevunique" query +const sqlIndexDataContinueBackwardStrictUnique = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id AND index_key < $index_key + ORDER BY index_key DESC, object_key ASC + LIMIT 1; +`; + +// continue a "prevunique" query +const sqlIndexDataContinueBackwardInclusiveUnique = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id AND index_key <= $index_key + ORDER BY index_key DESC, object_key ASC + LIMIT 1; +`; + +// continue a "next" query, no target object key +const sqlIndexDataContinueForwardStrictUnique = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id AND index_key > $index_key + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "next" query, no target object key +const sqlIndexDataContinueForwardInclusiveUnique = ` +SELECT index_key, object_key FROM index_data + WHERE index_id=$index_id AND index_key >= $index_key + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "next" query, strictly go to a further key +const sqlUniqueIndexDataContinueForwardStrict = ` +SELECT index_key, object_key FROM unique_index_data + WHERE index_id=$index_id AND index_key > $index_key + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "next" query, go to at least the specified key +const sqlUniqueIndexDataContinueForwardInclusive = ` +SELECT index_key, object_key FROM unique_index_data + WHERE index_id=$index_id AND index_key >= $index_key + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "prev" query +const sqlUniqueIndexDataContinueBackwardStrict = ` +SELECT index_key, object_key FROM unique_index_data + WHERE index_id=$index_id AND index_key < $index_key + ORDER BY index_key, object_key + LIMIT 1; +`; + +// continue a "prev" query +const sqlUniqueIndexDataContinueBackwardInclusive = ` +SELECT index_key, object_key FROM unique_index_data + WHERE index_id=$index_id AND index_key <= $index_key + ORDER BY index_key DESC, object_key DESC + LIMIT 1; +`; + +export interface SqliteBackendOptions { + filename: string; +} + +export async function createSqliteBackend( + sqliteImpl: Sqlite3Interface, + options: SqliteBackendOptions, +): Promise<SqliteBackend> { + const db = sqliteImpl.open(options.filename); + db.exec("PRAGMA foreign_keys = ON;"); + db.exec(schemaSql); + return new SqliteBackend(sqliteImpl, db); +} |