exchange

Base system with REST service to issue digital coins, run by the payment service provider
Log | Files | Refs | Submodules | README | LICENSE

commit 9a18a1242524ca371d185ca0bfc17b47ab9a9b89
parent 8c3e26a88043faf79f910862c8c2a3f1a1ecb912
Author: Christian Grothoff <christian@grothoff.org>
Date:   Wed,  9 Jul 2025 22:54:21 +0200

add UNIQUE constraint on h_payto+legitimization_serial for KYC attributes, do KYC upload all in one transaction, should fix #10158

Diffstat:
Msrc/exchange/taler-exchange-httpd_db.c | 18++++++++++--------
Msrc/exchange/taler-exchange-httpd_kyc-upload.c | 460++++++++++++++++++++++++++++++++++++++++++++++++++-----------------------------
Msrc/exchange/taler-exchange-httpd_metrics.c | 2++
Msrc/exchange/taler-exchange-httpd_metrics.h | 3++-
Asrc/exchangedb/0004-kyc_attributes.sql | 46++++++++++++++++++++++++++++++++++++++++++++++
Msrc/exchangedb/Makefile.am | 7+++++++
Asrc/exchangedb/exchange-0004.sql.in | 24++++++++++++++++++++++++
7 files changed, 385 insertions(+), 175 deletions(-)

diff --git a/src/exchange/taler-exchange-httpd_db.c b/src/exchange/taler-exchange-httpd_db.c @@ -151,10 +151,11 @@ TEH_DB_run_transaction (struct MHD_Connection *connection, { GNUNET_break (0); if (NULL != mhd_ret) - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_START_FAILED, - NULL); + *mhd_ret = TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_START_FAILED, + NULL); return GNUNET_SYSERR; } qs = cb (cb_cls, @@ -173,10 +174,11 @@ TEH_DB_run_transaction (struct MHD_Connection *connection, { TEH_plugin->rollback (TEH_plugin->cls); if (NULL != mhd_ret) - *mhd_ret = TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_COMMIT_FAILED, - NULL); + *mhd_ret = TALER_MHD_reply_with_error ( + connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_COMMIT_FAILED, + NULL); return GNUNET_SYSERR; } if (0 > qs) diff --git a/src/exchange/taler-exchange-httpd_kyc-upload.c b/src/exchange/taler-exchange-httpd_kyc-upload.c @@ -1,6 +1,6 @@ /* This file is part of TALER - Copyright (C) 2024 Taler Systems SA + Copyright (C) 2025 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software @@ -22,6 +22,7 @@ #include "taler-exchange-httpd_common_kyc.h" #include "taler-exchange-httpd_kyc-upload.h" +#define MAX_RETRIES 3 /** * Context used for processing the KYC upload req @@ -80,6 +81,21 @@ struct UploadContext */ const json_t *result; + /** + * Set by the transaction to the legitimization process row. + */ + uint64_t legi_process_row; + + /** + * Set by the transaction to the affected account payto hash. + */ + struct TALER_NormalizedPaytoHashP h_payto; + + /** + * Set by the transaction to true if the account is for a wallet. + */ + bool is_wallet; + }; @@ -182,6 +198,243 @@ aml_trigger_callback ( } +/** + * Do the main database transaction. + * + * @param cls closure with a `struct UploadContext` + * @param connection MHD request which triggered the transaction + * @param[out] mhd_ret set to MHD response status for @a connection, + * if transaction failed (!) + */ +static enum GNUNET_DB_QueryStatus +transact (void *cls, + struct MHD_Connection *connection, + MHD_RESULT *mhd_ret) +{ + struct UploadContext *uc = cls; + struct TEH_RequestContext *rc = uc->rc; + enum GNUNET_DB_QueryStatus qs; + json_t *jmeasures; + bool is_finished = false; + size_t enc_attributes_len; + void *enc_attributes; + const char *error_message; + char *form_name; + enum TALER_ErrorCode ec; + + qs = TEH_plugin->lookup_completed_legitimization ( + TEH_plugin->cls, + uc->legitimization_measure_serial_id, + uc->measure_index, + &uc->access_token, + &uc->h_payto, + &uc->is_wallet, + &jmeasures, + &is_finished, + &enc_attributes_len, + &enc_attributes); + /* FIXME: not exactly performant/elegant, should eventually + modify lookup_completed_legitimization to + return something if we are purely pending? */ + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "lookup_completed_legitimization"); + return qs; + case GNUNET_DB_STATUS_SOFT_ERROR: + return qs; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + qs = TEH_plugin->lookup_pending_legitimization ( + TEH_plugin->cls, + uc->legitimization_measure_serial_id, + &uc->access_token, + &uc->h_payto, + &jmeasures, + &is_finished, + &uc->is_wallet); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "lookup_pending_legitimization"); + return qs; + case GNUNET_DB_STATUS_SOFT_ERROR: + return qs; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_break_op (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_NOT_FOUND, + TALER_EC_EXCHANGE_KYC_CHECK_REQUEST_UNKNOWN, + NULL); + return GNUNET_DB_STATUS_HARD_ERROR; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + + if (NULL != enc_attributes) + { + json_t *xattributes; + + xattributes + = TALER_CRYPTO_kyc_attributes_decrypt ( + &TEH_attribute_key, + enc_attributes, + enc_attributes_len); + if (json_equal (xattributes, + uc->result)) + { + /* Request is idempotent! */ + json_decref (xattributes); + GNUNET_free (enc_attributes); + if (is_finished) + { + *mhd_ret = TALER_MHD_reply_static ( + rc->connection, + MHD_HTTP_NO_CONTENT, + NULL, + NULL, + 0); + return GNUNET_DB_STATUS_HARD_ERROR; + } + + /* Note: problem below is not here, but likely some previous + upload of the attributes failed badly in an AML program. */ + GNUNET_break (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_EXCHANGE_KYC_GENERIC_AML_LOGIC_BUG, + "attributes known, but legitimization process failed"); + return GNUNET_DB_STATUS_HARD_ERROR; + } + json_decref (xattributes); + GNUNET_free (enc_attributes); + /* Form was already done with with different attributes, conflict! */ + GNUNET_break_op (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_CONFLICT, + TALER_EC_EXCHANGE_KYC_FORM_ALREADY_UPLOADED, + NULL); + return GNUNET_DB_STATUS_HARD_ERROR; + } + if (is_finished) + { + /* This should not be possible (is_finished but NULL==enc_attributes), + but also we should not run logic again if we are finished. */ + GNUNET_break_op (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_CONFLICT, + TALER_EC_EXCHANGE_KYC_FORM_ALREADY_UPLOADED, + NULL); + return GNUNET_DB_STATUS_HARD_ERROR; + } + ec = TALER_KYCLOGIC_check_form (jmeasures, + uc->measure_index, + uc->result, + &form_name, + &error_message); + if (TALER_EC_NONE != ec) + { + GNUNET_break_op (0); + json_decref (jmeasures); + *mhd_ret = TALER_MHD_reply_with_ec ( + rc->connection, + ec, + error_message); + return GNUNET_DB_STATUS_HARD_ERROR; + } + json_decref (jmeasures); + + /* Setup KYC process (which we will then immediately 'finish') */ + qs = TEH_plugin->insert_kyc_requirement_process ( + TEH_plugin->cls, + &uc->h_payto, + uc->measure_index, + uc->legitimization_measure_serial_id, + form_name, + NULL, /* provider account ID */ + NULL, /* provider legi ID */ + &uc->legi_process_row); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + GNUNET_free (form_name); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_STORE_FAILED, + "insert_kyc_requirement_process"); + return GNUNET_DB_STATUS_HARD_ERROR; + case GNUNET_DB_STATUS_SOFT_ERROR: + GNUNET_free (form_name); + return qs; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_break (0); + GNUNET_free (form_name); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_INVARIANT_FAILURE, + "insert_kyc_requirement_process"); + return GNUNET_DB_STATUS_HARD_ERROR; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + qs = TEH_kyc_store_attributes ( + uc->legi_process_row, + &uc->h_payto, + form_name, + NULL /* provider account */, + NULL /* provider legi ID */, + GNUNET_TIME_UNIT_FOREVER_ABS, /* expiration time */ + uc->result); + GNUNET_free (form_name); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_STORE_FAILED, + "kyc_store_attributes"); + return GNUNET_DB_STATUS_HARD_ERROR; + case GNUNET_DB_STATUS_SOFT_ERROR: + return qs; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_break (0); + *mhd_ret = TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_STORE_FAILED, + "kyc_store_attributes"); + return GNUNET_DB_STATUS_HARD_ERROR; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + return qs; + } + GNUNET_assert (0); + *mhd_ret = MHD_NO; + return GNUNET_DB_STATUS_HARD_ERROR; +} + + MHD_RESULT TEH_handler_kyc_upload ( struct TEH_RequestContext *rc, @@ -252,174 +505,49 @@ TEH_handler_kyc_upload ( } + if (GNUNET_OK != + TEH_plugin->preflight (TEH_plugin->cls)) { - uint64_t legi_process_row; - struct TALER_NormalizedPaytoHashP h_payto; - enum GNUNET_DB_QueryStatus qs; - json_t *jmeasures; - bool is_finished = false; - size_t enc_attributes_len; - void *enc_attributes; - const char *error_message; - char *form_name; - enum TALER_ErrorCode ec; - bool is_wallet; - - qs = TEH_plugin->lookup_completed_legitimization ( - TEH_plugin->cls, - uc->legitimization_measure_serial_id, - uc->measure_index, - &uc->access_token, - &h_payto, - &is_wallet, - &jmeasures, - &is_finished, - &enc_attributes_len, - &enc_attributes); - /* FIXME: not exactly performant/elegant, should eventually - modify lookup_completed_legitimization to - return something if we are purely pending? */ - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - qs = TEH_plugin->lookup_pending_legitimization ( - TEH_plugin->cls, - uc->legitimization_measure_serial_id, - &uc->access_token, - &h_payto, - &jmeasures, - &is_finished, - &is_wallet); - if (qs < 0) - { - GNUNET_break (0); - return TALER_MHD_reply_with_error ( - rc->connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "lookup_pending_legitimization"); - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - GNUNET_break_op (0); - return TALER_MHD_reply_with_error ( - rc->connection, - MHD_HTTP_NOT_FOUND, - TALER_EC_EXCHANGE_KYC_CHECK_REQUEST_UNKNOWN, - NULL); - } + GNUNET_break (0); + return TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_SETUP_FAILED, + NULL); + } - if (is_finished) - { - if (NULL != enc_attributes) - { - json_t *xattributes; - - xattributes - = TALER_CRYPTO_kyc_attributes_decrypt ( - &TEH_attribute_key, - enc_attributes, - enc_attributes_len); - if (json_equal (xattributes, - uc->result)) - { - /* Request is idempotent! */ - json_decref (xattributes); - GNUNET_free (enc_attributes); - return TALER_MHD_reply_static ( - rc->connection, - MHD_HTTP_NO_CONTENT, - NULL, - NULL, - 0); - } - json_decref (xattributes); - GNUNET_free (enc_attributes); - } - /* Finished, and with no or different attributes, conflict! */ - GNUNET_break_op (0); - return TALER_MHD_reply_with_error ( - rc->connection, - MHD_HTTP_CONFLICT, - TALER_EC_EXCHANGE_KYC_FORM_ALREADY_UPLOADED, - NULL); - } - /* This _should_ not be possible (! is_finished but non-null enc_attributes), - but also cannot exactly hurt... */ - GNUNET_free (enc_attributes); - ec = TALER_KYCLOGIC_check_form (jmeasures, - uc->measure_index, - uc->result, - &form_name, - &error_message); - if (TALER_EC_NONE != ec) - { - GNUNET_break_op (0); - json_decref (jmeasures); - return TALER_MHD_reply_with_ec ( - rc->connection, - ec, - error_message); - } - json_decref (jmeasures); + { + MHD_RESULT mhd_ret = -1; - /* Setup KYC process (which we will then immediately 'finish') */ - qs = TEH_plugin->insert_kyc_requirement_process ( - TEH_plugin->cls, - &h_payto, - uc->measure_index, - uc->legitimization_measure_serial_id, - form_name, - NULL, /* provider account ID */ - NULL, /* provider legi ID */ - &legi_process_row); - if (qs <= 0) - { - GNUNET_break (0); - GNUNET_free (form_name); - return TALER_MHD_reply_with_error ( - rc->connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_STORE_FAILED, - "insert_kyc_requirement_process"); - } - qs = TEH_kyc_store_attributes ( - legi_process_row, - &h_payto, - form_name, - NULL /* provider account */, - NULL /* provider legi ID */, - GNUNET_TIME_UNIT_FOREVER_ABS, /* expiration time */ - uc->result); - GNUNET_free (form_name); - if (0 >= qs) - { - GNUNET_break (0); - return TALER_MHD_reply_with_error ( - rc->connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_STORE_FAILED, - "kyc_store_attributes"); - } + if (GNUNET_OK != + TEH_DB_run_transaction (rc->connection, + "kyc-upload", + TEH_MT_REQUEST_KYC_UPLOAD, + &mhd_ret, + &transact, + uc)) + return mhd_ret; + } - uc->kat = TEH_kyc_run_measure_for_attributes ( - &rc->async_scope_id, - legi_process_row, - &h_payto, - is_wallet, - &aml_trigger_callback, - uc); - if (NULL == uc->kat) - { - GNUNET_break (0); - return TALER_MHD_reply_with_error ( - rc->connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_EXCHANGE_KYC_GENERIC_AML_LOGIC_BUG, - "TEH_kyc_finished"); - } - MHD_suspend_connection (uc->rc->connection); - GNUNET_CONTAINER_DLL_insert (uc_head, - uc_tail, - uc); - return MHD_YES; + uc->kat = TEH_kyc_run_measure_for_attributes ( + &rc->async_scope_id, + uc->legi_process_row, + &uc->h_payto, + uc->is_wallet, + &aml_trigger_callback, + uc); + if (NULL == uc->kat) + { + GNUNET_break (0); + return TALER_MHD_reply_with_error ( + rc->connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_EXCHANGE_KYC_GENERIC_AML_LOGIC_BUG, + "TEH_kyc_finished"); } + MHD_suspend_connection (uc->rc->connection); + GNUNET_CONTAINER_DLL_insert (uc_head, + uc_tail, + uc); + return MHD_YES; } diff --git a/src/exchange/taler-exchange-httpd_metrics.c b/src/exchange/taler-exchange-httpd_metrics.c @@ -128,6 +128,8 @@ TEH_handler_metrics (struct TEH_RequestContext *rc, TEH_METRICS_num_requests[TEH_MT_REQUEST_IDEMPOTENT_DEPOSIT], "melt", TEH_METRICS_num_requests[TEH_MT_REQUEST_IDEMPOTENT_MELT], + "kyc-upload", + TEH_METRICS_num_requests[TEH_MT_REQUEST_KYC_UPLOAD], #endif "rsa", TEH_METRICS_num_signatures[TEH_MT_SIGNATURE_RSA], diff --git a/src/exchange/taler-exchange-httpd_metrics.h b/src/exchange/taler-exchange-httpd_metrics.h @@ -44,7 +44,8 @@ enum TEH_MetricTypeRequest TEH_MT_REQUEST_IDEMPOTENT_MELT = 10, TEH_MT_REQUEST_BATCH_DEPOSIT = 11, TEH_MT_REQUEST_POLICY_FULFILLMENT = 12, - TEH_MT_REQUEST_COUNT = 13 /* MUST BE LAST! */ + TEH_MT_REQUEST_KYC_UPLOAD = 13, + TEH_MT_REQUEST_COUNT = 14 /* MUST BE LAST! */ }; /** diff --git a/src/exchangedb/0004-kyc_attributes.sql b/src/exchangedb/0004-kyc_attributes.sql @@ -0,0 +1,46 @@ +-- +-- This file is part of TALER +-- Copyright (C) 2025 Taler Systems SA +-- +-- TALER is free software; you can redistribute it and/or modify it under the +-- terms of the GNU General Public License as published by the Free Software +-- Foundation; either version 3, or (at your option) any later version. +-- +-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY +-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +-- A PARTICULAR PURPOSE. See the GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License along with +-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> +-- + +CREATE OR REPLACE FUNCTION constrain_table_kyc_attributes4( + IN partition_suffix TEXT +) +RETURNS void +LANGUAGE plpgsql +AS $$ +DECLARE + table_name TEXT DEFAULT 'kyc_attributes'; +BEGIN + table_name = concat_ws('_', table_name, partition_suffix); + EXECUTE FORMAT ( + 'ALTER TABLE ' || table_name || + ' ADD CONSTRAINT ' || table_name || '_legitimization_serial ' + 'UNIQUE (h_payto,legitimization_serial)' + ); +END $$; + + +INSERT INTO exchange_tables + (name + ,version + ,action + ,partitioned + ,by_range) + VALUES + ('kyc_attributes4' + ,'exchange-0004' + ,'constrain' + ,TRUE + ,FALSE); diff --git a/src/exchangedb/Makefile.am b/src/exchangedb/Makefile.am @@ -38,6 +38,7 @@ sql_DATA = \ exchange-0001.sql \ exchange-0002.sql \ exchange-0003.sql \ + exchange-0004.sql \ drop.sql \ procedures.sql \ tops-0001.sql @@ -51,6 +52,7 @@ BUILT_SOURCES = \ CLEANFILES = \ exchange-0002.sql \ exchange-0003.sql \ + exchange-0004.sql \ procedures.sql procedures.sql: procedures.sql.in exchange_do_*.sql exchange_statistics_*.sql exchange_trigger_*.sql @@ -68,6 +70,11 @@ exchange-0003.sql: exchange-0003.sql.in 0003-*.sql gcc -E -P -undef - < exchange-0003.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@ chmod ugo-w $@ +exchange-0004.sql: exchange-0004.sql.in 0004-*.sql + chmod +w $@ 2> /dev/null || true + gcc -E -P -undef - < exchange-0004.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@ + chmod ugo-w $@ + check_SCRIPTS = \ test_idempotency.sh diff --git a/src/exchangedb/exchange-0004.sql.in b/src/exchangedb/exchange-0004.sql.in @@ -0,0 +1,24 @@ +-- +-- This file is part of TALER +-- Copyright (C) 2025 Taler Systems SA +-- +-- TALER is free software; you can redistribute it and/or modify it under the +-- terms of the GNU General Public License as published by the Free Software +-- Foundation; either version 3, or (at your option) any later version. +-- +-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY +-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +-- A PARTICULAR PURPOSE. See the GNU General Public License for more details. +-- +-- You should have received a copy of the GNU General Public License along with +-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> +-- + +BEGIN; + +SELECT _v.register_patch('exchange-0004', NULL, NULL); +SET search_path TO exchange; + +#include "0004-kyc_attributes.sql" + +COMMIT;