commit 1d1c847b793620acf3a2b193ab45eabf53234cb2
parent d0376d9e685a0e4920ec75b6e7ab176fa148aa4e
Author: Florian Dold <florian@dold.me>
Date: Tue, 8 Mar 2022 19:19:29 +0100
wallet: throttle all http requests
even from browsers / service workers
Diffstat:
7 files changed, 219 insertions(+), 186 deletions(-)
diff --git a/packages/taler-util/src/RequestThrottler.ts b/packages/taler-util/src/RequestThrottler.ts
@@ -0,0 +1,149 @@
+/*
+ This file is part of GNU Taler
+ (C) 2019 GNUnet e.V.
+
+ GNU Taler is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import { Logger } from "./logging.js";
+import { getTimestampNow, timestampCmp, timestampDifference } from "./time.js";
+
+/**
+ * Implementation of token bucket throttling.
+ */
+
+
+const logger = new Logger("RequestThrottler.ts");
+
+/**
+ * Maximum request per second, per origin.
+ */
+const MAX_PER_SECOND = 100;
+
+/**
+ * Maximum request per minute, per origin.
+ */
+const MAX_PER_MINUTE = 500;
+
+/**
+ * Maximum request per hour, per origin.
+ */
+const MAX_PER_HOUR = 2000;
+
+/**
+ * Throttling state for one origin.
+ */
+class OriginState {
+ tokensSecond: number = MAX_PER_SECOND;
+ tokensMinute: number = MAX_PER_MINUTE;
+ tokensHour: number = MAX_PER_HOUR;
+ private lastUpdate = getTimestampNow();
+
+ private refill(): void {
+ const now = getTimestampNow();
+ if (timestampCmp(now, this.lastUpdate) < 0) {
+ // Did the system time change?
+ this.lastUpdate = now;
+ return;
+ }
+ const d = timestampDifference(now, this.lastUpdate);
+ if (d.d_ms === "forever") {
+ throw Error("assertion failed");
+ }
+ this.tokensSecond = Math.min(
+ MAX_PER_SECOND,
+ this.tokensSecond + d.d_ms / 1000,
+ );
+ this.tokensMinute = Math.min(
+ MAX_PER_MINUTE,
+ this.tokensMinute + d.d_ms / 1000 / 60,
+ );
+ this.tokensHour = Math.min(
+ MAX_PER_HOUR,
+ this.tokensHour + d.d_ms / 1000 / 60 / 60,
+ );
+ this.lastUpdate = now;
+ }
+
+ /**
+ * Return true if the request for this origin should be throttled.
+ * Otherwise, take a token out of the respective buckets.
+ */
+ applyThrottle(): boolean {
+ this.refill();
+ if (this.tokensSecond < 1) {
+ logger.warn("request throttled (per second limit exceeded)");
+ return true;
+ }
+ if (this.tokensMinute < 1) {
+ logger.warn("request throttled (per minute limit exceeded)");
+ return true;
+ }
+ if (this.tokensHour < 1) {
+ logger.warn("request throttled (per hour limit exceeded)");
+ return true;
+ }
+ this.tokensSecond--;
+ this.tokensMinute--;
+ this.tokensHour--;
+ return false;
+ }
+}
+
+/**
+ * Request throttler, used as a "last layer of defense" when some
+ * other part of the re-try logic is broken and we're sending too
+ * many requests to the same exchange/bank/merchant.
+ */
+export class RequestThrottler {
+ private perOriginInfo: { [origin: string]: OriginState } = {};
+
+ /**
+ * Get the throttling state for an origin, or
+ * initialize if no state is associated with the
+ * origin yet.
+ */
+ private getState(origin: string): OriginState {
+ const s = this.perOriginInfo[origin];
+ if (s) {
+ return s;
+ }
+ const ns = (this.perOriginInfo[origin] = new OriginState());
+ return ns;
+ }
+
+ /**
+ * Apply throttling to a request.
+ *
+ * @returns whether the request should be throttled.
+ */
+ applyThrottle(requestUrl: string): boolean {
+ const origin = new URL(requestUrl).origin;
+ return this.getState(origin).applyThrottle();
+ }
+
+ /**
+ * Get the throttle statistics for a particular URL.
+ */
+ getThrottleStats(requestUrl: string): Record<string, unknown> {
+ const origin = new URL(requestUrl).origin;
+ const state = this.getState(origin);
+ return {
+ tokensHour: state.tokensHour,
+ tokensMinute: state.tokensMinute,
+ tokensSecond: state.tokensSecond,
+ maxTokensHour: MAX_PER_HOUR,
+ maxTokensMinute: MAX_PER_MINUTE,
+ maxTokensSecond: MAX_PER_SECOND,
+ };
+ }
+}
diff --git a/packages/taler-util/src/index.ts b/packages/taler-util/src/index.ts
@@ -30,3 +30,4 @@ export {
secretbox_open,
crypto_sign_keyPair_fromSeed,
} from "./nacl-fast.js";
+export { RequestThrottler } from "./RequestThrottler.js";
diff --git a/packages/taler-wallet-core/src/headless/NodeHttpLib.ts b/packages/taler-wallet-core/src/headless/NodeHttpLib.ts
@@ -25,7 +25,7 @@ import {
HttpRequestOptions,
HttpResponse,
} from "../util/http.js";
-import { RequestThrottler } from "../util/RequestThrottler.js";
+import { RequestThrottler } from "@gnu-taler/taler-util";
import Axios, { AxiosResponse } from "axios";
import { OperationFailedError, makeErrorDetails } from "../errors.js";
import { Logger, bytesToString } from "@gnu-taler/taler-util";
diff --git a/packages/taler-wallet-core/src/util/RequestThrottler.ts b/packages/taler-wallet-core/src/util/RequestThrottler.ts
@@ -1,156 +0,0 @@
-/*
- This file is part of GNU Taler
- (C) 2019 GNUnet e.V.
-
- GNU Taler is free software; you can redistribute it and/or modify it under the
- terms of the GNU General Public License as published by the Free Software
- Foundation; either version 3, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along with
- GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
- */
-
-/**
- * Implementation of token bucket throttling.
- */
-
-/**
- * Imports.
- */
-import {
- getTimestampNow,
- timestampDifference,
- timestampCmp,
- Logger,
- URL,
-} from "@gnu-taler/taler-util";
-
-const logger = new Logger("RequestThrottler.ts");
-
-/**
- * Maximum request per second, per origin.
- */
-const MAX_PER_SECOND = 100;
-
-/**
- * Maximum request per minute, per origin.
- */
-const MAX_PER_MINUTE = 500;
-
-/**
- * Maximum request per hour, per origin.
- */
-const MAX_PER_HOUR = 2000;
-
-/**
- * Throttling state for one origin.
- */
-class OriginState {
- tokensSecond: number = MAX_PER_SECOND;
- tokensMinute: number = MAX_PER_MINUTE;
- tokensHour: number = MAX_PER_HOUR;
- private lastUpdate = getTimestampNow();
-
- private refill(): void {
- const now = getTimestampNow();
- if (timestampCmp(now, this.lastUpdate) < 0) {
- // Did the system time change?
- this.lastUpdate = now;
- return;
- }
- const d = timestampDifference(now, this.lastUpdate);
- if (d.d_ms === "forever") {
- throw Error("assertion failed");
- }
- this.tokensSecond = Math.min(
- MAX_PER_SECOND,
- this.tokensSecond + d.d_ms / 1000,
- );
- this.tokensMinute = Math.min(
- MAX_PER_MINUTE,
- this.tokensMinute + d.d_ms / 1000 / 60,
- );
- this.tokensHour = Math.min(
- MAX_PER_HOUR,
- this.tokensHour + d.d_ms / 1000 / 60 / 60,
- );
- this.lastUpdate = now;
- }
-
- /**
- * Return true if the request for this origin should be throttled.
- * Otherwise, take a token out of the respective buckets.
- */
- applyThrottle(): boolean {
- this.refill();
- if (this.tokensSecond < 1) {
- logger.warn("request throttled (per second limit exceeded)");
- return true;
- }
- if (this.tokensMinute < 1) {
- logger.warn("request throttled (per minute limit exceeded)");
- return true;
- }
- if (this.tokensHour < 1) {
- logger.warn("request throttled (per hour limit exceeded)");
- return true;
- }
- this.tokensSecond--;
- this.tokensMinute--;
- this.tokensHour--;
- return false;
- }
-}
-
-/**
- * Request throttler, used as a "last layer of defense" when some
- * other part of the re-try logic is broken and we're sending too
- * many requests to the same exchange/bank/merchant.
- */
-export class RequestThrottler {
- private perOriginInfo: { [origin: string]: OriginState } = {};
-
- /**
- * Get the throttling state for an origin, or
- * initialize if no state is associated with the
- * origin yet.
- */
- private getState(origin: string): OriginState {
- const s = this.perOriginInfo[origin];
- if (s) {
- return s;
- }
- const ns = (this.perOriginInfo[origin] = new OriginState());
- return ns;
- }
-
- /**
- * Apply throttling to a request.
- *
- * @returns whether the request should be throttled.
- */
- applyThrottle(requestUrl: string): boolean {
- const origin = new URL(requestUrl).origin;
- return this.getState(origin).applyThrottle();
- }
-
- /**
- * Get the throttle statistics for a particular URL.
- */
- getThrottleStats(requestUrl: string): Record<string, unknown> {
- const origin = new URL(requestUrl).origin;
- const state = this.getState(origin);
- return {
- tokensHour: state.tokensHour,
- tokensMinute: state.tokensMinute,
- tokensSecond: state.tokensSecond,
- maxTokensHour: MAX_PER_HOUR,
- maxTokensMinute: MAX_PER_MINUTE,
- maxTokensSecond: MAX_PER_SECOND,
- };
- }
-}
diff --git a/packages/taler-wallet-core/tsconfig.json b/packages/taler-wallet-core/tsconfig.json
@@ -21,7 +21,7 @@
"esModuleInterop": true,
"importHelpers": true,
"rootDir": "./src",
- "typeRoots": ["./node_modules/@types"],
+ "typeRoots": ["./node_modules/@types"]
},
"references": [
{
diff --git a/packages/taler-wallet-webextension/src/browserHttpLib.ts b/packages/taler-wallet-webextension/src/browserHttpLib.ts
@@ -24,7 +24,11 @@ import {
HttpResponse,
Headers,
} from "@gnu-taler/taler-wallet-core";
-import { Logger, TalerErrorCode } from "@gnu-taler/taler-util";
+import {
+ Logger,
+ RequestThrottler,
+ TalerErrorCode,
+} from "@gnu-taler/taler-util";
const logger = new Logger("browserHttpLib");
@@ -33,12 +37,32 @@ const logger = new Logger("browserHttpLib");
* browser's XMLHttpRequest.
*/
export class BrowserHttpLib implements HttpRequestLibrary {
- fetch(url: string, options?: HttpRequestOptions): Promise<HttpResponse> {
- const method = options?.method ?? "GET";
+ private throttle = new RequestThrottler();
+ private throttlingEnabled = true;
+
+ fetch(
+ requestUrl: string,
+ options?: HttpRequestOptions,
+ ): Promise<HttpResponse> {
+ const requestMethod = options?.method ?? "GET";
let requestBody = options?.body;
+
+ if (this.throttlingEnabled && this.throttle.applyThrottle(requestUrl)) {
+ const parsedUrl = new URL(requestUrl);
+ throw OperationFailedError.fromCode(
+ TalerErrorCode.WALLET_HTTP_REQUEST_THROTTLED,
+ `request to origin ${parsedUrl.origin} was throttled`,
+ {
+ requestMethod,
+ requestUrl,
+ throttleStats: this.throttle.getThrottleStats(requestUrl),
+ },
+ );
+ }
+
return new Promise<HttpResponse>((resolve, reject) => {
const myRequest = new XMLHttpRequest();
- myRequest.open(method, url);
+ myRequest.open(requestMethod, requestUrl);
if (options?.headers) {
for (const headerName in options.headers) {
myRequest.setRequestHeader(headerName, options.headers[headerName]);
@@ -58,7 +82,7 @@ export class BrowserHttpLib implements HttpRequestLibrary {
TalerErrorCode.WALLET_NETWORK_ERROR,
"Could not make request",
{
- requestUrl: url,
+ requestUrl: requestUrl,
},
),
);
@@ -71,7 +95,7 @@ export class BrowserHttpLib implements HttpRequestLibrary {
TalerErrorCode.WALLET_NETWORK_ERROR,
"HTTP request failed (status 0, maybe URI scheme was wrong?)",
{
- requestUrl: url,
+ requestUrl: requestUrl,
},
);
reject(exc);
@@ -92,7 +116,7 @@ export class BrowserHttpLib implements HttpRequestLibrary {
TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE,
"Invalid JSON from HTTP response",
{
- requestUrl: url,
+ requestUrl: requestUrl,
httpStatusCode: myRequest.status,
},
);
@@ -102,7 +126,7 @@ export class BrowserHttpLib implements HttpRequestLibrary {
TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE,
"Invalid JSON from HTTP response",
{
- requestUrl: url,
+ requestUrl: requestUrl,
httpStatusCode: myRequest.status,
},
);
@@ -126,10 +150,10 @@ export class BrowserHttpLib implements HttpRequestLibrary {
headerMap.set(headerName, value);
});
const resp: HttpResponse = {
- requestUrl: url,
+ requestUrl: requestUrl,
status: myRequest.status,
headers: headerMap,
- requestMethod: method,
+ requestMethod: requestMethod,
json: makeJson,
text: makeText,
bytes: async () => myRequest.response,
diff --git a/packages/taler-wallet-webextension/src/serviceWorkerHttpLib.ts b/packages/taler-wallet-webextension/src/serviceWorkerHttpLib.ts
@@ -17,37 +17,55 @@
/**
* Imports.
*/
-import { Logger, TalerErrorCode } from "@gnu-taler/taler-util";
+import { RequestThrottler, TalerErrorCode } from "@gnu-taler/taler-util";
import {
- Headers, HttpRequestLibrary,
+ Headers,
+ HttpRequestLibrary,
HttpRequestOptions,
HttpResponse,
- OperationFailedError
+ OperationFailedError,
} from "@gnu-taler/taler-wallet-core";
-const logger = new Logger("browserHttpLib");
-
/**
* An implementation of the [[HttpRequestLibrary]] using the
* browser's XMLHttpRequest.
*/
export class ServiceWorkerHttpLib implements HttpRequestLibrary {
- async fetch(requestUrl: string, options?: HttpRequestOptions): Promise<HttpResponse> {
+ private throttle = new RequestThrottler();
+ private throttlingEnabled = true;
+
+ async fetch(
+ requestUrl: string,
+ options?: HttpRequestOptions,
+ ): Promise<HttpResponse> {
const requestMethod = options?.method ?? "GET";
const requestBody = options?.body;
const requestHeader = options?.headers;
+ if (this.throttlingEnabled && this.throttle.applyThrottle(requestUrl)) {
+ const parsedUrl = new URL(requestUrl);
+ throw OperationFailedError.fromCode(
+ TalerErrorCode.WALLET_HTTP_REQUEST_THROTTLED,
+ `request to origin ${parsedUrl.origin} was throttled`,
+ {
+ requestMethod,
+ requestUrl,
+ throttleStats: this.throttle.getThrottleStats(requestUrl),
+ },
+ );
+ }
+
const response = await fetch(requestUrl, {
headers: requestHeader,
body: requestBody,
method: requestMethod,
// timeout: options?.timeout
- })
+ });
const headerMap = new Headers();
response.headers.forEach((value, key) => {
headerMap.set(key, value);
- })
+ });
return {
headers: headerMap,
status: response.status,
@@ -56,11 +74,9 @@ export class ServiceWorkerHttpLib implements HttpRequestLibrary {
json: makeJsonHandler(response, requestUrl),
text: makeTextHandler(response, requestUrl),
bytes: async () => (await response.blob()).arrayBuffer(),
- }
-
+ };
}
-
get(url: string, opt?: HttpRequestOptions): Promise<HttpResponse> {
return this.fetch(url, {
method: "GET",
@@ -89,7 +105,7 @@ function makeTextHandler(response: Response, requestUrl: string) {
return async function getJsonFromResponse(): Promise<any> {
let respText;
try {
- respText = await response.text()
+ respText = await response.text();
} catch (e) {
throw OperationFailedError.fromCode(
TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE,
@@ -100,15 +116,15 @@ function makeTextHandler(response: Response, requestUrl: string) {
},
);
}
- return respText
- }
+ return respText;
+ };
}
function makeJsonHandler(response: Response, requestUrl: string) {
return async function getJsonFromResponse(): Promise<any> {
let responseJson;
try {
- responseJson = await response.json()
+ responseJson = await response.json();
} catch (e) {
throw OperationFailedError.fromCode(
TalerErrorCode.WALLET_RECEIVED_MALFORMED_RESPONSE,
@@ -129,7 +145,6 @@ function makeJsonHandler(response: Response, requestUrl: string) {
},
);
}
- return responseJson
- }
+ return responseJson;
+ };
}
-
