commit b3401596a3ef38220da7a4c94ea4bfe10aca71b0
parent 68b2c69b2eb15280768db1fc828bcd30b360d9a1
Author: Christian Grothoff <christian@grothoff.org>
Date: Thu, 3 Jul 2025 11:20:48 +0200
fix #9284: load compressed files from disk instead of doing compression at runtime for the SPAs
Diffstat:
12 files changed, 209 insertions(+), 140 deletions(-)
diff --git a/debian/taler-auditor.postinst b/debian/taler-auditor.postinst
@@ -29,6 +29,13 @@ configure)
mark_secret /etc/taler-auditor/secrets/auditor-db.secret.conf
mark_secret /etc/taler-auditor/secrets/exchange-accountcredentials-1.secret.conf
+ # Try to generate compressed versions of the SPA
+ for n in index.css index.js index.html
+ do
+ TDIR="/usr/share/taler-auditor/"
+ gzip --best - < "${TDIR}/spa/$n" > "${TDIR}/spa/$n.gz" || true
+ zstd -19 - < "${TDIR}/spa/$n" > "${TDIR}/spa/$n.zstd" || true
+ done
;;
abort-upgrade | abort-remove | abort-deconfigure) ;;
diff --git a/debian/taler-exchange.postinst b/debian/taler-exchange.postinst
@@ -95,6 +95,15 @@ configure)
/etc/taler-exchange/secrets/exchange-db.secret.conf
fi
+ # Try to generate compressed versions of the SPAs
+ for n in forms.json index.css index.js index.html
+ do
+ TDIR="/usr/share/taler-exchange/"
+ gzip --best - < "${TDIR}/aml-spa/$n" > "${TDIR}/aml-spa/$n.gz" || true
+ gzip --best - < "${TDIR}/kyc-spa/$n" > "${TDIR}/kyc-spa/$n.gz" || true
+ zstd -19 - < "${TDIR}/aml-spa/$n" > "${TDIR}/aml-spa/$n.zstd" || true
+ zstd -19 - < "${TDIR}/kyc-spa/$n" > "${TDIR}/kyc-spa/$n.zstd" || true
+ done
;;
abort-upgrade | abort-remove | abort-deconfigure) ;;
diff --git a/src/auditordb/0002-auditor_reserve_not_closed_inconsistency.sql b/src/auditordb/0002-auditor_reserve_not_closed_inconsistency.sql
@@ -16,7 +16,7 @@
CREATE TABLE IF NOT EXISTS auditor_reserve_not_closed_inconsistency
(
- row_id BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE ,
+ row_id BIGINT GENERATED BY DEFAULT AS IDENTITY UNIQUE,
reserve_pub BYTEA PRIMARY KEY NOT NULL CHECK (LENGTH(reserve_pub)=32),
balance taler_amount NOT NULL,
expiration_time BIGINT,
diff --git a/src/auditordb/pg_insert_early_aggregation.c b/src/auditordb/pg_insert_early_aggregation.c
@@ -44,7 +44,7 @@ TAH_PG_insert_early_aggregation (
PREPARE (pg,
"auditor_insert_early_aggregation",
- "INSERT INTO auditor_pending_deposits "
+ "INSERT INTO auditor_early_aggregations"
"(batch_deposit_serial_id"
",tracking_serial_id"
",amount"
diff --git a/src/auditordb/pg_insert_pending_deposit.c b/src/auditordb/pg_insert_pending_deposit.c
@@ -52,7 +52,8 @@ TAH_PG_insert_pending_deposit (
",batch_deposit_serial_id"
",deadline"
") VALUES ($1,$2,$3,$4);");
- return GNUNET_PQ_eval_prepared_non_select (pg->conn,
- "auditor_insert_pending_deposit",
- params);
+ return GNUNET_PQ_eval_prepared_non_select (
+ pg->conn,
+ "auditor_insert_pending_deposit",
+ params);
}
diff --git a/src/auditordb/pg_insert_reserve_not_closed_inconsistency.c b/src/auditordb/pg_insert_reserve_not_closed_inconsistency.c
@@ -41,7 +41,8 @@ TAH_PG_insert_reserve_not_closed_inconsistency (
" balance,"
" expiration_time,"
" diagnostic"
- ") VALUES ($1,$2,$3,$4);"
+ ") VALUES ($1,$2,$3,$4)"
+ " ON CONFLICT DO NOTHING;"
);
return GNUNET_PQ_eval_prepared_non_select (pg->conn,
"auditor_reserve_not_closed_inconsistency_insert",
diff --git a/src/exchange/taler-exchange-httpd_keys.c b/src/exchange/taler-exchange-httpd_keys.c
@@ -4050,12 +4050,14 @@ TEH_keys_get_handler (struct TEH_RequestContext *rc,
&setup_general_response_headers,
ksh);
- return MHD_queue_response (rc->connection,
- MHD_HTTP_OK,
- (MHD_YES ==
- TALER_MHD_can_compress (rc->connection))
- ? krd->response_compressed
- : krd->response_uncompressed);
+ return MHD_queue_response (
+ rc->connection,
+ MHD_HTTP_OK,
+ (TALER_MHD_CT_DEFLATE ==
+ TALER_MHD_can_compress (rc->connection,
+ TALER_MHD_CT_DEFLATE))
+ ? krd->response_compressed
+ : krd->response_uncompressed);
}
}
diff --git a/src/include/taler/taler_mhd_lib.h b/src/include/taler/taler_mhd_lib.h
@@ -136,13 +136,48 @@ TALER_MHD_body_compress (void **buf,
/**
- * Is HTTP body deflate compression supported by the client?
+ * List of compression types we check for. Larger numeric values
+ * indicate a preferred algorithm.
+ */
+enum TALER_MHD_CompressionType
+{
+ /**
+ * Compression is not supported.
+ */
+ TALER_MHD_CT_NONE = 0,
+
+ /**
+ * Deflate compression supported.
+ */
+ TALER_MHD_CT_DEFLATE,
+
+ /**
+ * gzip compression supported.
+ */
+ TALER_MHD_CT_GZIP,
+
+ /**
+ * zstd compression supported.
+ */
+ TALER_MHD_CT_ZSTD,
+
+ /**
+ * End of list marker.
+ */
+ TALER_MHD_CT_MAX
+};
+
+
+/**
+ * What type of HTTP compression is supported by the client?
*
* @param connection connection to check
+ * @param max maximum compression level to check for
* @return #MHD_YES if 'deflate' compression is allowed
*/
-MHD_RESULT
-TALER_MHD_can_compress (struct MHD_Connection *connection);
+enum TALER_MHD_CompressionType
+TALER_MHD_can_compress (struct MHD_Connection *connection,
+ enum TALER_MHD_CompressionType max);
/**
diff --git a/src/mhd/mhd_legal.c b/src/mhd/mhd_legal.c
@@ -280,8 +280,9 @@ TALER_MHD_reply_legal (struct MHD_Connection *conn,
return_t:
/* try to compress the response */
resp = NULL;
- if ( (MHD_YES ==
- TALER_MHD_can_compress (conn)) &&
+ if ( (TALER_MHD_CT_DEFLATE ==
+ TALER_MHD_can_compress (conn,
+ TALER_MHD_CT_DEFLATE)) &&
(NULL != t->compressed_terms) )
{
resp = MHD_create_response_from_buffer (t->compressed_terms_size,
diff --git a/src/mhd/mhd_responses.c b/src/mhd/mhd_responses.c
@@ -67,34 +67,53 @@ TALER_MHD_add_global_headers (struct MHD_Response *response,
}
-MHD_RESULT
-TALER_MHD_can_compress (struct MHD_Connection *connection)
+enum TALER_MHD_CompressionType
+TALER_MHD_can_compress (struct MHD_Connection *connection,
+ enum TALER_MHD_CompressionType max)
{
+ struct NameMap
+ {
+ const char *name;
+ enum TALER_MHD_CompressionType ct;
+ } map[] = {
+ /* sorted largest to smallest on purpose! */
+ { "zstd", TALER_MHD_CT_ZSTD },
+ { "gzip", TALER_MHD_CT_GZIP },
+ { "deflate", TALER_MHD_CT_DEFLATE },
+ { NULL, TALER_MHD_CT_NONE }
+ };
const char *ae;
const char *de;
if (0 != (TM_go & TALER_MHD_GO_DISABLE_COMPRESSION))
- return MHD_NO;
+ return TALER_MHD_CT_NONE;
ae = MHD_lookup_connection_value (connection,
MHD_HEADER_KIND,
MHD_HTTP_HEADER_ACCEPT_ENCODING);
if (NULL == ae)
- return MHD_NO;
+ return TALER_MHD_CT_NONE;
if (0 == strcmp (ae,
"*"))
return MHD_YES;
- de = strstr (ae,
- "deflate");
- if (NULL == de)
- return MHD_NO;
- if ( ( (de == ae) ||
- (de[-1] == ',') ||
- (de[-1] == ' ') ) &&
- ( (de[strlen ("deflate")] == '\0') ||
- (de[strlen ("deflate")] == ',') ||
- (de[strlen ("deflate")] == ';') ) )
- return MHD_YES;
- return MHD_NO;
+ for (unsigned int i=0; NULL != map[i].name; i++)
+ {
+ const char *name = map[i].name;
+
+ if (map[i].ct > max)
+ continue; /* not allowed by client */
+ de = strstr (ae,
+ name);
+ if (NULL == de)
+ continue;
+ if ( ( (de == ae) ||
+ (de[-1] == ',') ||
+ (de[-1] == ' ') ) &&
+ ( (de[strlen (name)] == '\0') ||
+ (de[strlen (name)] == ',') ||
+ (de[strlen (name)] == ';') ) )
+ return map[i].ct;
+ }
+ return TALER_MHD_CT_NONE;
}
@@ -198,8 +217,9 @@ TALER_MHD_reply_json (struct MHD_Connection *connection,
json_len = strlen (json_str);
/* try to compress the body */
is_compressed = MHD_NO;
- if (MHD_YES ==
- TALER_MHD_can_compress (connection))
+ if (TALER_MHD_CT_DEFLATE ==
+ TALER_MHD_can_compress (connection,
+ TALER_MHD_CT_DEFLATE))
is_compressed = TALER_MHD_body_compress (&json_str,
&json_len);
response = MHD_create_response_from_buffer (json_len,
diff --git a/src/mhd/mhd_spa.c b/src/mhd/mhd_spa.c
@@ -46,14 +46,9 @@ struct WebuiFile
char *path;
/**
- * SPA resource, compressed.
+ * SPA resource, deflate compressed.
*/
- struct MHD_Response *zresponse;
-
- /**
- * SPA resource, vanilla.
- */
- struct MHD_Response *response;
+ struct MHD_Response *responses[TALER_MHD_CT_MAX];
};
@@ -81,6 +76,7 @@ TALER_MHD_spa_handler (const struct TALER_MHD_Spa *spa,
const char *path)
{
struct WebuiFile *w = NULL;
+ enum TALER_MHD_CompressionType comp;
if ( (NULL == path) ||
(0 == strcmp (path,
@@ -95,20 +91,21 @@ TALER_MHD_spa_handler (const struct TALER_MHD_Spa *spa,
w = pos;
break;
}
- if (NULL == w)
+ if ( (NULL == w) ||
+ (NULL == w->responses[TALER_MHD_CT_NONE]) )
return TALER_MHD_reply_with_error (connection,
MHD_HTTP_NOT_FOUND,
TALER_EC_GENERIC_ENDPOINT_UNKNOWN,
path);
- if ( (MHD_YES ==
- TALER_MHD_can_compress (connection)) &&
- (NULL != w->zresponse) )
+ comp = TALER_MHD_can_compress (connection,
+ TALER_MHD_CT_MAX - 1);
+ if (NULL != w->responses[comp])
return MHD_queue_response (connection,
MHD_HTTP_OK,
- w->zresponse);
+ w->responses[comp]);
return MHD_queue_response (connection,
MHD_HTTP_OK,
- w->response);
+ w->responses[TALER_MHD_CT_NONE]);
}
@@ -163,11 +160,20 @@ build_webui (void *cls,
struct TALER_MHD_Spa *spa = cls;
int fd;
struct stat sb;
- struct MHD_Response *zresponse = NULL;
struct MHD_Response *response;
- const char *ext;
+ char *ext;
const char *mime;
+ const char *slash;
+ char *fn;
+ const char *cts = NULL;
+ enum TALER_MHD_CompressionType ct = TALER_MHD_CT_NONE;
+ slash = strrchr (dn, '/');
+ if (NULL == slash)
+ {
+ GNUNET_break (0);
+ return GNUNET_SYSERR;
+ }
fd = open (dn,
O_RDONLY);
if (-1 == fd)
@@ -188,14 +194,40 @@ build_webui (void *cls,
return GNUNET_SYSERR;
}
- mime = NULL;
- ext = strrchr (dn, '.');
+ fn = GNUNET_strdup (slash + 1);
+ ext = strrchr (fn,
+ '.');
+ if (NULL == ext)
+ {
+ GNUNET_break (0 == close (fd));
+ GNUNET_free (fn);
+ return GNUNET_OK;
+ }
+ if (0 == strcmp (ext,
+ ".gz"))
+ {
+ cts = "gzip";
+ ct = TALER_MHD_CT_GZIP;
+ *ext = '\0';
+ ext = strrchr (fn, '.');
+ }
+ if (0 == strcmp (ext,
+ ".zstd"))
+ {
+ cts = "zstd";
+ ct = TALER_MHD_CT_ZSTD;
+ *ext = '\0';
+ ext = strrchr (fn, '.');
+ }
if (NULL == ext)
{
GNUNET_break (0 == close (fd));
+ GNUNET_free (fn);
return GNUNET_OK;
}
ext++;
+
+ mime = NULL;
for (unsigned int i = 0; NULL != mime_map[i].ext; i++)
if (0 == strcasecmp (ext,
mime_map[i].ext))
@@ -204,98 +236,58 @@ build_webui (void *cls,
break;
}
+ response = MHD_create_response_from_fd (
+ sb.st_size,
+ fd /* FD now owned by MHD! */);
+ if (NULL == response)
{
- void *in;
- ssize_t r;
- size_t csize;
-
- in = GNUNET_malloc_large (sb.st_size);
- if (NULL == in)
- {
- GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
- "malloc");
- GNUNET_break (0 == close (fd));
- return GNUNET_SYSERR;
- }
- r = read (fd,
- in,
- sb.st_size);
- if ( (-1 == r) ||
- (sb.st_size != (size_t) r) )
- {
- GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
- "read",
- dn);
- GNUNET_free (in);
- GNUNET_break (0 == close (fd));
- return GNUNET_SYSERR;
- }
- csize = (size_t) r;
- if (MHD_YES ==
- TALER_MHD_body_compress (&in,
- &csize))
- {
- zresponse = MHD_create_response_from_buffer (csize,
- in,
- MHD_RESPMEM_MUST_FREE);
- if (NULL != zresponse)
- {
- if (MHD_NO ==
- MHD_add_response_header (zresponse,
- MHD_HTTP_HEADER_CONTENT_ENCODING,
- "deflate"))
- {
- GNUNET_break (0);
- MHD_destroy_response (zresponse);
- zresponse = NULL;
- }
- if (NULL != mime)
- GNUNET_break (MHD_YES ==
- MHD_add_response_header (zresponse,
- MHD_HTTP_HEADER_CONTENT_TYPE,
- mime));
- }
- }
- else
- {
- GNUNET_free (in);
- }
+ GNUNET_free (fn);
+ return GNUNET_SYSERR;
}
-
- response = MHD_create_response_from_fd (sb.st_size,
- fd);
- if (NULL == response)
+ if ( (NULL != cts) &&
+ (MHD_NO ==
+ MHD_add_response_header (response,
+ MHD_HTTP_HEADER_CONTENT_ENCODING,
+ cts)) )
{
- GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
- "open",
- dn);
- GNUNET_break (0 == close (fd));
- if (NULL != zresponse)
- {
- MHD_destroy_response (zresponse);
- zresponse = NULL;
- }
+ GNUNET_break (0);
+ MHD_destroy_response (response);
+ GNUNET_free (fn);
return GNUNET_SYSERR;
}
if (NULL != mime)
+ {
GNUNET_break (MHD_YES ==
MHD_add_response_header (response,
MHD_HTTP_HEADER_CONTENT_TYPE,
mime));
+ }
{
struct WebuiFile *w;
- const char *fn;
- fn = strrchr (dn, '/');
- GNUNET_assert (NULL != fn);
- w = GNUNET_new (struct WebuiFile);
- w->path = GNUNET_strdup (fn + 1);
- w->response = response;
- w->zresponse = zresponse;
- GNUNET_CONTAINER_DLL_insert (spa->webui_head,
- spa->webui_tail,
- w);
+ for (w = spa->webui_head;
+ NULL != w;
+ w = w->next)
+ {
+ if (0 == strcmp (fn,
+ w->path))
+ break;
+ }
+ if (NULL == w)
+ {
+ w = GNUNET_new (struct WebuiFile);
+ w->path = fn;
+ GNUNET_CONTAINER_DLL_insert (spa->webui_head,
+ spa->webui_tail,
+ w);
+ }
+ else
+ {
+ GNUNET_free (fn);
+ }
+ GNUNET_assert (NULL == w->responses[ct]);
+ w->responses[ct] = response;
}
return GNUNET_OK;
}
@@ -352,15 +344,15 @@ TALER_MHD_spa_free (struct TALER_MHD_Spa *spa)
GNUNET_CONTAINER_DLL_remove (spa->webui_head,
spa->webui_tail,
w);
- if (NULL != w->response)
+ for (enum TALER_MHD_CompressionType ct = TALER_MHD_CT_NONE;
+ ct < TALER_MHD_CT_MAX;
+ ct++)
{
- MHD_destroy_response (w->response);
- w->response = NULL;
- }
- if (NULL != w->zresponse)
- {
- MHD_destroy_response (w->zresponse);
- w->zresponse = NULL;
+ if (NULL != w->responses[ct])
+ {
+ MHD_destroy_response (w->responses[ct]);
+ w->responses[ct] = NULL;
+ }
}
GNUNET_free (w->path);
GNUNET_free (w);
diff --git a/src/templating/templating_api.c b/src/templating/templating_api.c
@@ -294,8 +294,9 @@ TALER_TEMPLATING_build (struct MHD_Connection *connection,
{
bool compressed = false;
- if (MHD_YES ==
- TALER_MHD_can_compress (connection))
+ if (TALER_MHD_CT_DEFLATE ==
+ TALER_MHD_can_compress (connection,
+ TALER_MHD_CT_DEFLATE))
{
compressed = TALER_MHD_body_compress ((void **) &body,
&body_size);
