From bce6a266ed4e273e40c90ecf68d7e1444f211526 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 3 May 2023 20:18:37 +0200 Subject: get new taler-merchant-exchange helper to build --- src/backend/.gitignore | 1 + src/backend/Makefile.am | 18 + src/backend/taler-merchant-exchange.c | 604 ++++++++++++++++++++++------- src/backenddb/Makefile.am | 1 + src/backenddb/merchant-0005.sql | 13 +- src/backenddb/pg_update_transfer_status.c | 65 ++++ src/backenddb/pg_update_transfer_status.h | 51 +++ src/backenddb/plugin_merchantdb_postgres.c | 3 + src/include/taler_merchant_testing_lib.h | 13 +- src/include/taler_merchantdb_plugin.h | 22 ++ src/testing/Makefile.am | 1 + src/testing/test_kyc_api.c | 4 + src/testing/test_merchant_api.c | 4 + src/testing/testing_api_cmd_tme.c | 161 ++++++++ 14 files changed, 815 insertions(+), 146 deletions(-) create mode 100644 src/backenddb/pg_update_transfer_status.c create mode 100644 src/backenddb/pg_update_transfer_status.h create mode 100644 src/testing/testing_api_cmd_tme.c (limited to 'src') diff --git a/src/backend/.gitignore b/src/backend/.gitignore index 0154aa73..62ef6e0a 100644 --- a/src/backend/.gitignore +++ b/src/backend/.gitignore @@ -1,2 +1,3 @@ taler-merchant-webhook taler-merchant-wirewatch +taler-merchant-exchange diff --git a/src/backend/Makefile.am b/src/backend/Makefile.am index 9da9164e..4edca6ac 100644 --- a/src/backend/Makefile.am +++ b/src/backend/Makefile.am @@ -16,6 +16,7 @@ EXTRA_DIST = \ $(pkgcfg_DATA) bin_PROGRAMS = \ + taler-merchant-exchange \ taler-merchant-httpd \ taler-merchant-webhook \ taler-merchant-wirewatch @@ -155,6 +156,23 @@ taler_merchant_httpd_CFLAGS = \ $(AM_CFLAGS) +taler_merchant_exchange_SOURCES = \ + taler-merchant-exchange.c +taler_merchant_exchange_LDADD = \ + $(top_builddir)/src/backenddb/libtalermerchantdb.la \ + -ltalerexchange \ + -ltalerjson \ + -ltalerutil \ + -ltalerpq \ + -lgnunetjson \ + -lgnunetcurl \ + -lgnunetutil \ + -lcurl \ + $(XLIB) +taler_merchant_exchange_CFLAGS = \ + $(AM_CFLAGS) + + taler_merchant_webhook_SOURCES = \ taler-merchant-webhook.c taler_merchant_webhook_LDADD = \ diff --git a/src/backend/taler-merchant-exchange.c b/src/backend/taler-merchant-exchange.c index 66893f03..ff480ca9 100644 --- a/src/backend/taler-merchant-exchange.c +++ b/src/backend/taler-merchant-exchange.c @@ -45,6 +45,13 @@ */ #define EXCHANGE_INQUIRY_LIMIT 16 + +/** + * Information about an inquiry job. + */ +struct Inquiry; + + /** * Information about an exchange. */ @@ -60,6 +67,16 @@ struct Exchange */ struct Exchange *prev; + /** + * Head of active inquiries. + */ + struct Inquiry *w_head; + + /** + * Tail of active inquiries. + */ + struct Inquiry *w_tail; + /** * Which exchange are we tracking here. */ @@ -90,6 +107,12 @@ struct Exchange */ struct GNUNET_TIME_Relative retry_delay; + /** + * How long should we wait between requests + * for transfer details? + */ + struct GNUNET_TIME_Relative transfer_delay; + /** * False to indicate that there is an ongoing * /keys transfer we are waiting for; @@ -120,6 +143,11 @@ struct Inquiry */ struct Exchange *exchange; + /** + * Task where we retry fetching transfer details from the exchange. + */ + struct GNUNET_SCHEDULER_Task *task; + /** * For which merchant instance is this tracking request? */ @@ -135,12 +163,6 @@ struct Inquiry */ struct TALER_EXCHANGE_TransfersGetHandle *wdh; - /** - * Pointer to the detail that we are currently - * checking in #check_transfer(). - */ - const struct TALER_TrackTransferDetails *current_detail; - /** * When did the transfer happen? */ @@ -161,24 +183,6 @@ struct Inquiry */ uint64_t rowid; - /** - * Which transaction detail are we currently looking at? - */ - unsigned int current_offset; - - /** - * #GNUNET_NO if we did not find a matching coin. - * #GNUNET_SYSERR if we found a matching coin, but the amounts do not match. - * #GNUNET_OK if we did find a matching coin. - */ - enum GNUNET_GenericReturnValue check_transfer_result; - - /** - * Are we done with the exchange request for this - * inquiry? - */ - bool exchange_done; - }; @@ -192,16 +196,6 @@ static struct Exchange *e_head; */ static struct Exchange *e_tail; -/** - * Head of active inquiries. - */ -static struct Inquiry *w_head; - -/** - * Tail of active inquiries. - */ -static struct Inquiry *w_tail; - /** * The merchant's configuration. */ @@ -244,14 +238,17 @@ static unsigned int active_inquiries; static int global_ret; /** - * How many transactions should we fetch at most per batch? + * #GNUNET_YES if we are in test mode and should exit when idle. */ -static unsigned int batch_size = 32; +static int test_mode; /** - * #GNUNET_YES if we are in test mode and should exit when idle. + * True if the last DB query was limited by the + * #OPEN_INQUIRY_LIMIT and we thus should check again + * as soon as we are substantially below that limit, + * and not only when we get a DB notification. */ -static int test_mode; +static bool at_limit; /** @@ -267,27 +264,21 @@ exchange_request (void *cls); * The exchange @a e is ready to handle more inquiries, * prepare to launch them. * - * @param e exchange to potentially launch inquiries on + * @param[in,out] e exchange to potentially launch inquiries on */ static void -launch_inquiries_at_exchange (const struct Exchange *e) +launch_inquiries_at_exchange (struct Exchange *e) { - /* Note: this is an O(n) that should be optimized to an O(1) by tracking - inquiries per exchange at the exchange... */ - for (struct Inquiry w = w_head; + for (struct Inquiry *w = e->w_head; NULL != w; w = w->next) { - if (w->exchange_done) - continue; - if (w->exchange != e) - continue; + if (e->exchange_inquiries > EXCHANGE_INQUIRY_LIMIT) + break; if ( (NULL == w->task) && (NULL == w->wdh) ) { - /* Note: we should additionally count the number of exchange - requests launched here to not then run into the per-exchange - limit at exchange_request */ + e->exchange_inquiries++; w->task = GNUNET_SCHEDULER_add_now (&exchange_request, w); } @@ -295,39 +286,94 @@ launch_inquiries_at_exchange (const struct Exchange *e) } +/** + * Function that initiates a /keys download. + * + * @param cls a `struct Exchange *` + */ +static void +download_keys (void *cls); + + /** * Function called with information about who is auditing * a particular exchange and what keys the exchange is using. * * @param cls closure with a `struct Exchange *` - * @param hr HTTP response data - * @param keys information about the various keys used - * by the exchange, NULL if /keys failed - * @param compat protocol compatibility information + * @param kr response data */ static void cert_cb ( void *cls, - const struct TALER_EXCHANGE_HttpResponse *hr, - const struct TALER_EXCHANGE_Keys *keys, - enum TALER_EXCHANGE_VersionCompatibility compat) + const struct TALER_EXCHANGE_KeysResponse *kr) { struct Exchange *e = cls; + struct GNUNET_TIME_Timestamp t; + struct GNUNET_TIME_Absolute n; - switch (hr->http_status) + switch (kr->hr.http_status) { case MHD_HTTP_OK: e->ready = true; launch_inquiries_at_exchange (e); - // FIXME: schedule retry? + /* Reset back-off */ + e->retry_delay = GNUNET_TIME_UNIT_ZERO; + /* Success: rate limit at once per minute */ + e->first_retry = GNUNET_TIME_relative_to_absolute ( + GNUNET_TIME_UNIT_MINUTES); + /* Moreover usually only go after the current + response actually expired */ + t = TALER_EXCHANGE_check_keys_current (e->conn, + TALER_EXCHANGE_CKF_NONE); + n = GNUNET_TIME_absolute_max (t.abs_time, + e->first_retry); + if (NULL != e->retry_task) + GNUNET_SCHEDULER_cancel (e->retry_task); + e->retry_task = GNUNET_SCHEDULER_add_at (n, + &download_keys, + e); break; default: - // FIXME: schedule retry! + e->retry_delay + = GNUNET_TIME_STD_BACKOFF (e->retry_delay); + e->first_retry + = GNUNET_TIME_relative_to_absolute (e->retry_delay); + + if (NULL != e->retry_task) + GNUNET_SCHEDULER_cancel (e->retry_task); + e->retry_task = GNUNET_SCHEDULER_add_delayed (e->retry_delay, + &download_keys, + e); break; } } +static void +download_keys (void *cls) +{ + struct Exchange *e = cls; + struct GNUNET_TIME_Relative n; + + /* If we do not hear back again soon, try again automatically */ + n = GNUNET_TIME_STD_BACKOFF (e->retry_delay); + n = GNUNET_TIME_relative_max (n, + GNUNET_TIME_UNIT_MINUTES); + e->retry_task = GNUNET_SCHEDULER_add_delayed (n, + &download_keys, + e); + if (NULL == e->conn) + e->conn = TALER_EXCHANGE_connect (ctx, + e->exchange_url, + &cert_cb, + e, + TALER_EXCHANGE_OPTION_END); + else + (void) TALER_EXCHANGE_check_keys_current (e->conn, + TALER_EXCHANGE_CKF_NONE); +} + + /** * Updates the transaction status for inquiry @a w to the given values. * @@ -344,7 +390,22 @@ update_transaction_status (const struct Inquiry *w, bool failed, bool verified) { - // FIXME: DB update here. Log result, on failure shutdown. + enum GNUNET_DB_QueryStatus qs; + + qs = db_plugin->update_transfer_status (db_plugin->cls, + w->exchange->exchange_url, + &w->wtid, + next_attempt, + ec, + failed, + verified); + if (qs < 0) + { + GNUNET_break (0); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } } @@ -370,15 +431,21 @@ find_exchange (const char *exchange_url) GNUNET_CONTAINER_DLL_insert (e_head, e_tail, e); - e->conn = TALER_EXCHANGE_connect (ctx, - exchange_url, - &cert_cb, - e, - TALER_EXCHANGE_OPTION_END); + e->retry_task = GNUNET_SCHEDULER_add_now (&download_keys, + e); return e; } +/** + * Finds new transfers that require work in the merchant database. + * + * @param cls NULL + */ +static void +find_work (void *cls); + + /** * Free resources of @a w. * @@ -387,6 +454,8 @@ find_exchange (const char *exchange_url) static void end_inquiry (struct Inquiry *w) { + struct Exchange *e = w->exchange; + GNUNET_assert (active_inquiries > 0); active_inquiries--; if (NULL != w->wdh) @@ -396,10 +465,18 @@ end_inquiry (struct Inquiry *w) } GNUNET_free (w->instance_id); GNUNET_free (w->payto_uri); - GNUNET_CONTAINER_DLL_remove (w_head, - w_tail, + GNUNET_CONTAINER_DLL_remove (e->w_head, + e->w_tail, w); GNUNET_free (w); + if ( (active_inquiries < OPEN_INQUIRY_LIMIT / 2) && + (NULL == task) && + (at_limit) ) + { + at_limit = false; + task = GNUNET_SCHEDULER_add_now (&find_work, + NULL); + } } @@ -414,16 +491,16 @@ shutdown_task (void *cls) (void) cls; GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Running shutdown\n"); - while (NULL != w_head) - { - struct Inquiry *w = w_head; - - end_inquiry (w); - } while (NULL != e_head) { struct Exchange *e = e_head; + while (NULL != e->w_head) + { + struct Inquiry *w = e->w_head; + + end_inquiry (w); + } GNUNET_free (e->exchange_url); TALER_EXCHANGE_disconnect (e->conn); GNUNET_CONTAINER_DLL_remove (e_head, @@ -452,26 +529,27 @@ shutdown_task (void *cls) } -// FIXME: where is this used? Where do we check the totals!? /** - * Check that the given @a wire_fee is what the @a exchange_pub should charge + * Check that the given @a wire_fee is what the @a e should charge * at the @a execution_time. If the fee is correct (according to our * database), return #GNUNET_OK. If we do not have the fee structure in our * DB, we just accept it and return #GNUNET_NO; if we have proof that the fee * is bogus, we respond with the proof to the client and return * #GNUNET_SYSERR. * - * @param ptc context of the transfer to respond to + * @param w inquiry to check fees of * @param execution_time time of the wire transfer * @param wire_fee fee claimed by the exchange * @return #GNUNET_SYSERR if we returned hard proof of * missbehavior from the exchange to the client */ static enum GNUNET_GenericReturnValue -check_wire_fee (struct PostTransfersContext *ptc, +check_wire_fee (struct Inquiry *w, struct GNUNET_TIME_Timestamp execution_time, const struct TALER_Amount *wire_fee) { + struct Exchange *e = w->exchange; + const struct TALER_EXCHANGE_Keys *keys; struct TALER_WireFeeSet fees; struct TALER_MasterSignatureP master_sig; struct GNUNET_TIME_Timestamp start_date; @@ -479,35 +557,39 @@ check_wire_fee (struct PostTransfersContext *ptc, enum GNUNET_DB_QueryStatus qs; char *wire_method; - wire_method = TALER_payto_get_method (ptc->payto_uri); - qs = TMH_db->lookup_wire_fee (TMH_db->cls, - &ptc->master_pub, - wire_method, - execution_time, - &fees, - &start_date, - &end_date, - &master_sig); + keys = TALER_EXCHANGE_get_keys (e->conn); + if (NULL == keys) + { + GNUNET_break (0); + return GNUNET_NO; + } + wire_method = TALER_payto_get_method (w->payto_uri); + qs = db_plugin->lookup_wire_fee (db_plugin->cls, + &keys->master_pub, + wire_method, + execution_time, + &fees, + &start_date, + &end_date, + &master_sig); switch (qs) { case GNUNET_DB_STATUS_HARD_ERROR: GNUNET_break (0); - ptc->response_code = MHD_HTTP_INTERNAL_SERVER_ERROR; - ptc->response = TALER_MHD_make_error (TALER_EC_GENERIC_DB_FETCH_FAILED, - "lookup_wire_fee"); + GNUNET_free (wire_method); return GNUNET_SYSERR; case GNUNET_DB_STATUS_SOFT_ERROR: - ptc->soft_retry = true; + GNUNET_free (wire_method); return GNUNET_NO; case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to find wire fee for `%s' and method `%s' at %s in DB, accepting blindly that the fee is %s\n", - TALER_B2S (&ptc->master_pub), + TALER_B2S (&keys->master_pub), wire_method, GNUNET_TIME_timestamp2s (execution_time), TALER_amount2s (wire_fee)); GNUNET_free (wire_method); - return GNUNET_NO; + return GNUNET_OK; case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: break; } @@ -517,64 +599,179 @@ check_wire_fee (struct PostTransfersContext *ptc, GNUNET_free (wire_method); return GNUNET_OK; /* expected_fee >= wire_fee */ } - /* Wire fee check failed, export proof to auditor! */ - // FIXME: report error! GNUNET_free (wire_method); return GNUNET_SYSERR; } +/** + * Closure for #check_transfer() + */ +struct CheckTransferContext +{ + + /** + * Pointer to the detail that we are currently + * checking in #check_transfer(). + */ + const struct TALER_TrackTransferDetails *current_detail; + + /** + * Which transaction detail are we currently looking at? + */ + unsigned int current_offset; + + /** + * #GNUNET_NO if we did not find a matching coin. + * #GNUNET_SYSERR if we found a matching coin, but the amounts do not match. + * #GNUNET_OK if we did find a matching coin. + */ + enum GNUNET_GenericReturnValue check_transfer_result; + + /** + * Set to error code, if any. + */ + enum TALER_ErrorCode ec; + + /** + * Set to true if @e ec indicates a permanent failure. + */ + bool failure; +}; + + +/** + * This function checks that the information about the coin which + * was paid back by _this_ wire transfer matches what _we_ (the merchant) + * knew about this coin. + * + * @param cls closure with our `struct CheckTransferContext *` + * @param exchange_url URL of the exchange that issued @a coin_pub + * @param amount_with_fee amount the exchange will transfer for this coin + * @param deposit_fee fee the exchange will charge for this coin + * @param refund_fee fee the exchange will charge for refunding this coin + * @param wire_fee paid wire fee + * @param h_wire hash of merchant's wire details + * @param deposit_timestamp when did the exchange receive the deposit + * @param refund_deadline until when are refunds allowed + * @param exchange_sig signature by the exchange + * @param exchange_pub exchange signing key used for @a exchange_sig + */ +static void +check_transfer (void *cls, + const char *exchange_url, + const struct TALER_Amount *amount_with_fee, + const struct TALER_Amount *deposit_fee, + const struct TALER_Amount *refund_fee, + const struct TALER_Amount *wire_fee, + const struct TALER_MerchantWireHashP *h_wire, + struct GNUNET_TIME_Timestamp deposit_timestamp, + struct GNUNET_TIME_Timestamp refund_deadline, + const struct TALER_ExchangeSignatureP *exchange_sig, + const struct TALER_ExchangePublicKeyP *exchange_pub) +{ + struct CheckTransferContext *ctc = cls; + const struct TALER_TrackTransferDetails *ttd = ctc->current_detail; + + if (GNUNET_SYSERR == ctc->check_transfer_result) + { + GNUNET_break (0); + return; /* already had a serious issue; odd that we're called more than once as well... */ + } + if ( (0 != TALER_amount_cmp (amount_with_fee, + &ttd->coin_value)) || + (0 != TALER_amount_cmp (deposit_fee, + &ttd->coin_fee)) ) + { + /* Disagreement between the exchange and us about how much this + coin is worth! */ + GNUNET_break_op (0); + ctc->check_transfer_result = GNUNET_SYSERR; + /* Build the `TrackTransferConflictDetails` */ + ctc->ec = TALER_EC_MERCHANT_PRIVATE_POST_TRANSFERS_CONFLICTING_REPORTS; + ctc->failure = true; + /* TODO: this should be reported to the auditor! */ + return; + } + ctc->check_transfer_result = GNUNET_OK; +} + + /** * Function called with detailed wire transfer data, including all * of the coin transactions that were combined into the wire transfer. * * @param cls closure a `struct Inquiry *` - * @param hr HTTP response details - * @param td transfer data + * @param tgr response details */ static void wire_transfer_cb (void *cls, - const struct TALER_EXCHANGE_HttpResponse *hr, - const struct TALER_EXCHANGE_TransferData *td) + const struct TALER_EXCHANGE_TransfersGetResponse *tgr) { struct Inquiry *w = cls; struct Exchange *e = w->exchange; enum GNUNET_DB_QueryStatus qs; + const struct TALER_EXCHANGE_TransferData *td = NULL; e->exchange_inquiries--; + w->wdh = NULL; if (EXCHANGE_INQUIRY_LIMIT - 1 == e->exchange_inquiries) launch_inquiries_at_exchange (e); - w->wdh = NULL; GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Got response code %u from exchange for GET /transfers/$WTID\n", - hr->http_status); - switch (hr->http_status) + tgr->hr.http_status); + switch (tgr->hr.http_status) { case MHD_HTTP_OK: + td = &tgr->details.ok.td; + e->transfer_delay = GNUNET_TIME_UNIT_ZERO; break; + case MHD_HTTP_BAD_REQUEST: + case MHD_HTTP_FORBIDDEN: + update_transaction_status (w, + GNUNET_TIME_UNIT_FOREVER_ABS, + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_HARD_FAILURE, + true, + false); + return; case MHD_HTTP_NOT_FOUND: update_transaction_status (w, GNUNET_TIME_UNIT_FOREVER_ABS, - TALER_EC_MERCHANT_XXX, + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_FATAL_NOT_FOUND, true, false); return; + case MHD_HTTP_INTERNAL_SERVER_ERROR: + case MHD_HTTP_BAD_GATEWAY: + case MHD_HTTP_GATEWAY_TIMEOUT: + e->transfer_delay = GNUNET_TIME_STD_BACKOFF (e->transfer_delay); + update_transaction_status (w, + GNUNET_TIME_relative_to_absolute ( + e->transfer_delay), + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_TRANSIENT_FAILURE, + false, + false); + return; default: + e->transfer_delay = GNUNET_TIME_STD_BACKOFF (e->transfer_delay); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Unexpected HTTP status %u\n", + tgr->hr.http_status); update_transaction_status (w, - GNUNET_TIME_relative_to_absolute (delay), - TALER_EC_MERCHANT_XXX, + GNUNET_TIME_relative_to_absolute ( + e->transfer_delay), + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_TRANSIENT_FAILURE, false, false); return; } - TMH_db->preflight (TMH_db->cls); - /* Ok, exchange answer is acceptable, store it */ - qs = TMH_db->insert_transfer_details (TMH_db->cls, - w->instance_id, - w->exchange_url, - w->payto_uri, - &w->wtid, - td); + db_plugin->preflight (db_plugin->cls); + qs = db_plugin->insert_transfer_details (db_plugin->cls, + w->instance_id, + w->exchange->exchange_url, + w->payto_uri, + &w->wtid, + td); if (0 > qs) { /* Always report on DB error as well to enable diagnostics */ @@ -585,18 +782,113 @@ wire_transfer_cb (void *cls, } if (0 == qs) { - GNUNET_break (0); + /* FIXME: set as confirmed!!? */ + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Transfer already known. Ignoring duplicate.\n"); + /* FIXME: if not verified & not ultimately failed, + we probably should still try to verify! */ return; } - // FIXME: check wire fee somewhere!??! + + { + struct CheckTransferContext ctc = { + .ec = TALER_EC_NONE, + .failure = false + }; + + for (unsigned int i = 0; idetails_length; i++) + { + const struct TALER_TrackTransferDetails *ttd = &td->details[i]; + enum GNUNET_DB_QueryStatus qs; + + if (TALER_EC_NONE != ctc.ec) + break; /* already encountered an error */ + ctc.current_offset = i; + ctc.current_detail = ttd; + /* Set the coin as "never seen" before. */ + ctc.check_transfer_result = GNUNET_NO; + qs = db_plugin->lookup_deposits_by_contract_and_coin ( + db_plugin->cls, + w->instance_id, + &ttd->h_contract_terms, + &ttd->coin_pub, + &check_transfer, + &ctc); + switch (qs) + { + case GNUNET_DB_STATUS_SOFT_ERROR: + GNUNET_break (0); + ctc.ec = TALER_EC_GENERIC_DB_FETCH_FAILED; + break; + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + ctc.ec = TALER_EC_GENERIC_DB_FETCH_FAILED; + break; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + /* The exchange says we made this deposit, but WE do not + recall making it (corrupted / unreliable database?)! + Well, let's say thanks and accept the money! */ + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Failed to find payment data in DB\n"); + ctc.check_transfer_result = GNUNET_OK; + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + switch (ctc.check_transfer_result) + { + case GNUNET_NO: + /* Internal error: how can we have called #check_transfer() + but still have no result? */ + GNUNET_break (0); + ctc.ec = TALER_EC_GENERIC_INTERNAL_INVARIANT_FAILURE; + return; + case GNUNET_SYSERR: + /* #check_transfer() failed, report conflict! */ + GNUNET_break_op (0); + GNUNET_assert (TALER_EC_NONE != ctc.ec); + return; + case GNUNET_OK: + break; + } + } + if (TALER_EC_NONE != ctc.ec) + { + update_transaction_status ( + w, + ctc.failure + ? GNUNET_TIME_UNIT_FOREVER_ABS + : GNUNET_TIME_relative_to_absolute ( + GNUNET_TIME_UNIT_MINUTES), + ctc.ec, + ctc.failure, + false); + return; + } + } + + if (GNUNET_SYSERR == + check_wire_fee (w, + td->execution_time, + &td->wire_fee)) + { + GNUNET_break_op (0); + update_transaction_status (w, + GNUNET_TIME_UNIT_FOREVER_ABS, + TALER_EC_MERCHANT_PRIVATE_POST_TRANSFERS_BAD_WIRE_FEE, + true, + false); + return; + } + if (0 != TALER_amount_cmp (&td->total_amount, &w->total)) { - /* record inconsistency in DB! (TODO: report to auditor!?) */ + GNUNET_break_op (0); update_transaction_status (w, GNUNET_TIME_UNIT_FOREVER_ABS, - TALER_EC_MERCHANT_XXX, + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_CONFLICTING_TRANSFERS, true, false); return; @@ -622,8 +914,6 @@ exchange_request (void *cls) struct Exchange *e = w->exchange; GNUNET_assert (e->ready); - if (EXCHANGE_INQUIRY_LIMIT <= e->exchange_inquiries) - return; /* blocked by exchange rate limit */ w->wdh = TALER_EXCHANGE_transfers_get (e->conn, &w->wtid, &wire_transfer_cb, @@ -631,17 +921,21 @@ exchange_request (void *cls) if (NULL == w->wdh) { GNUNET_break (0); + e->exchange_inquiries--; + e->transfer_delay = GNUNET_TIME_STD_BACKOFF (e->transfer_delay); update_transaction_status (w, - GNUNET_TIME_relative_to_absolute (delay), - TALER_EC_MERCHANT_XXX, + GNUNET_TIME_relative_to_absolute ( + e->transfer_delay), + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_TRANSIENT_FAILURE, false, false); return; } - e->exchange_inquiries++; + /* Wait at least 1m for the network transfer */ update_transaction_status (w, - GNUNET_TIME_relative_to_absolute (delay), - TALER_EC_MERCHANT_EXCHANGE_GET_TRANSFER_IN_PROGRESS, + GNUNET_TIME_relative_to_absolute ( + GNUNET_TIME_UNIT_MINUTES), + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_AWAITING_LIST, false, false); } @@ -671,9 +965,11 @@ start_inquiry ( const struct TALER_Amount *total, struct GNUNET_TIME_Absolute next_attempt) { + struct Exchange *e; struct Inquiry *w; (void) cls; + e = find_exchange (exchange_url); active_inquiries++; w = GNUNET_new (struct Inquiry); w->payto_uri = GNUNET_strdup (payto_uri); @@ -681,37 +977,42 @@ start_inquiry ( w->rowid = rowid; w->wtid = *wtid; w->total = *total; - GNUNET_CONTAINER_DLL_insert (w_head, - w_tail, + GNUNET_CONTAINER_DLL_insert (e->w_head, + e->w_tail, w); - w->exchange = find_exchange (exchange_url); + w->exchange = e; if (w->exchange->ready) w->task = GNUNET_SCHEDULER_add_now (&exchange_request, w); + /* Wait at least 1 minute for /keys */ update_transaction_status (w, - GNUNET_TIME_relative_to_absolute (delay), - TALER_EC_MERCHANT_EXCHANGE_KEYS_IN_PROGRESS, + GNUNET_TIME_relative_to_absolute ( + GNUNET_TIME_UNIT_MINUTES), + TALER_EC_MERCHANT_EXCHANGE_TRANSFERS_AWAITING_KEYS, false, false); } -/** - * Finds new transfers that require work in the merchant - * database. - * - * @param cls NULL - */ static void find_work (void *cls) { enum GNUNET_DB_QueryStatus qs; + int limit; + (void) cls; task = NULL; - // NOTE: SELECT WHERE confirmed AND NOT verified AND NOT failed?; - // FIXME: use LIMIT clause! => When do we try again if LIMIT applied? + GNUNET_assert (OPEN_INQUIRY_LIMIT >= active_inquiries); + limit = OPEN_INQUIRY_LIMIT - active_inquiries; + if (0 == limit) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Not looking for work: at limit\n"); + at_limit = true; + return; + } qs = db_plugin->select_open_transfers (db_plugin->cls, - OPEN_INQUIRY_LIMIT - active_inquiries, + limit, &start_inquiry, NULL); if (qs < 0) @@ -721,6 +1022,25 @@ find_work (void *cls) GNUNET_SCHEDULER_shutdown (); return; } + if (qs == limit) + { + /* DB limited response, re-trigger DB interaction + the moment we significantly fall below the + limit */ + at_limit = true; + } + if (0 == active_inquiries) + { + if (test_mode) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No more open inquiries and in test mode. Existing.\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "No open inquiries found, waiting for notification to resume\n"); + } } @@ -740,6 +1060,12 @@ transfer_added (void *cls, (void) cls; (void) extra; (void) extra_size; + if (active_inquiries > OPEN_INQUIRY_LIMIT / 2) + { + /* Trigger DB only once we are substantially below the limit */ + at_limit = true; + return; + } if (NULL != task) return; task = GNUNET_SCHEDULER_add_now (&find_work, @@ -795,7 +1121,7 @@ run (void *cls, { struct GNUNET_DB_EventHeaderP es = { .size = htons (sizeof (es)), - .type = htons (TALER_DBEVENT_MERCHANT_XXX) + .type = htons (TALER_DBEVENT_MERCHANT_WIRE_TRANSFER_CONFIRMED) }; eh = db_plugin->event_listen (db_plugin->cls, diff --git a/src/backenddb/Makefile.am b/src/backenddb/Makefile.am index 2debd00e..2ed54ef7 100644 --- a/src/backenddb/Makefile.am +++ b/src/backenddb/Makefile.am @@ -59,6 +59,7 @@ libtaler_plugin_merchantdb_postgres_la_SOURCES = \ pg_select_open_transfers.h pg_select_open_transfers.c \ pg_lookup_instances.h pg_lookup_instances.c \ pg_lookup_transfers.h pg_lookup_transfers.c \ + pg_update_transfer_status.h pg_update_transfer_status.c \ pg_delete_exchange_accounts.h pg_delete_exchange_accounts.c \ pg_select_accounts_by_exchange.h pg_select_accounts_by_exchange.c \ pg_insert_exchange_account.h pg_insert_exchange_account.c \ diff --git a/src/backenddb/merchant-0005.sql b/src/backenddb/merchant-0005.sql index 5c01e55b..a0e283fa 100644 --- a/src/backenddb/merchant-0005.sql +++ b/src/backenddb/merchant-0005.sql @@ -35,6 +35,7 @@ ALTER TABLE merchant_tip_reserve_keys ADD COLUMN master_pub BYTEA NOT NULL CHECK (LENGTH(master_pub)=32); ALTER TABLE merchant_transfers + ADD COLUMN ready_time INT8 NOT NULL DEFAULT (0), ADD COLUMN failed BOOLEAN NOT NULL DEFAULT FALSE, ADD COLUMN validation_status INT4 DEFAULT NULL; COMMENT ON COLUMN merchant_transfers.failed @@ -42,12 +43,12 @@ COMMENT ON COLUMN merchant_transfers.failed COMMENT ON COLUMN merchant_transfers.validation_status IS 'Taler error code describing the state of the validation'; ---CREATE INDEX merchant_transfers_by_open --- ON merchant_transfers --- (ready_time ASC) --- WHERE confirmed AND NOT (failed OR verified); ---COMMENT ON INDEX merchant_transfers_by_open --- IS 'For select_open_transfers'; +CREATE INDEX merchant_transfers_by_open + ON merchant_transfers + (ready_time ASC) + WHERE confirmed AND NOT (failed OR verified); +COMMENT ON INDEX merchant_transfers_by_open + IS 'For select_open_transfers'; ALTER TABLE merchant_accounts diff --git a/src/backenddb/pg_update_transfer_status.c b/src/backenddb/pg_update_transfer_status.c new file mode 100644 index 00000000..9898984d --- /dev/null +++ b/src/backenddb/pg_update_transfer_status.c @@ -0,0 +1,65 @@ +/* + This file is part of TALER + Copyright (C) 2022 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + TALER; see the file COPYING. If not, see + */ +/** + * @file backenddb/pg_update_transfer_status.c + * @brief Implementation of the update_transfer_status function for Postgres + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include +#include +#include "pg_update_transfer_status.h" +#include "pg_helper.h" + + +enum GNUNET_DB_QueryStatus +TMH_PG_update_transfer_status ( + void *cls, + const char *exchange_url, + const struct TALER_WireTransferIdentifierRawP *wtid, + struct GNUNET_TIME_Absolute next_attempt, + enum TALER_ErrorCode ec, + bool failed, + bool verified) +{ + struct PostgresClosure *pg = cls; + uint32_t ec32 = (uint32_t) ec; + struct GNUNET_PQ_QueryParam params[] = { + GNUNET_PQ_query_param_auto_from_type (wtid), + GNUNET_PQ_query_param_string (exchange_url), + GNUNET_PQ_query_param_uint32 (&ec32), + GNUNET_PQ_query_param_bool (failed), + GNUNET_PQ_query_param_bool (verified), + GNUNET_PQ_query_param_absolute_time (&next_attempt), + GNUNET_PQ_query_param_end + }; + + check_connection (pg); + PREPARE (pg, + "update_transfer_status", + "UPDATE merchant_transfers SET" + " validation_status=$3" + ",failed=$4" + ",verified=$5" + ",ready_time=$6" + " WHERE wtid=$1" + " AND exchange_url=$2"); + return GNUNET_PQ_eval_prepared_non_select ( + pg->conn, + "update_transfer_status", + params); +} diff --git a/src/backenddb/pg_update_transfer_status.h b/src/backenddb/pg_update_transfer_status.h new file mode 100644 index 00000000..2828b25e --- /dev/null +++ b/src/backenddb/pg_update_transfer_status.h @@ -0,0 +1,51 @@ +/* + This file is part of TALER + Copyright (C) 2022 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + TALER; see the file COPYING. If not, see + */ +/** + * @file backenddb/pg_update_transfer_status.h + * @brief implementation of the update_transfer_status function for Postgres + * @author Christian Grothoff + */ +#ifndef PG_UPDATE_TRANSFER_STATUS_H +#define PG_UPDATE_TRANSFER_STATUS_H + +#include +#include +#include "taler_merchantdb_plugin.h" + + +/** + * Update transfer status. + * + * @param cls closure + * @param exchange_url the exchange that made the transfer + * @param wtid wire transfer subject + * @param next_attempt when should we try again (if ever) + * @param ec current error state of checking the transfer + * @param failed true if validation has failed for good + * @param verified true if validation has succeeded for good + * @return database transaction status + */ +enum GNUNET_DB_QueryStatus +TMH_PG_update_transfer_status ( + void *cls, + const char *exchange_url, + const struct TALER_WireTransferIdentifierRawP *wtid, + struct GNUNET_TIME_Absolute next_attempt, + enum TALER_ErrorCode ec, + bool failed, + bool verified); + +#endif diff --git a/src/backenddb/plugin_merchantdb_postgres.c b/src/backenddb/plugin_merchantdb_postgres.c index 72ebd7e4..ca26c01d 100644 --- a/src/backenddb/plugin_merchantdb_postgres.c +++ b/src/backenddb/plugin_merchantdb_postgres.c @@ -41,6 +41,7 @@ #include "pg_select_accounts_by_exchange.h" #include "pg_insert_exchange_account.h" #include "pg_lookup_reserves.h" +#include "pg_update_transfer_status.h" /** @@ -9504,6 +9505,8 @@ libtaler_plugin_merchantdb_postgres_init (void *cls) plugin->update_instance_auth = &postgres_update_instance_auth; plugin->activate_account = &postgres_activate_account; plugin->inactivate_account = &postgres_inactivate_account; + plugin->update_transfer_status + = &TMH_PG_update_transfer_status; plugin->lookup_products = &postgres_lookup_products; plugin->lookup_product = &postgres_lookup_product; plugin->delete_product = &postgres_delete_product; diff --git a/src/include/taler_merchant_testing_lib.h b/src/include/taler_merchant_testing_lib.h index c860baf2..372dd6ee 100644 --- a/src/include/taler_merchant_testing_lib.h +++ b/src/include/taler_merchant_testing_lib.h @@ -1782,7 +1782,7 @@ TALER_TESTING_cmd_merchant_delete_webhook (const char *label, unsigned int http_status); /** - * to use the 'taler-merchant-webhook' program. + * Command to run the 'taler-merchant-webhook' program. * * @param label command label. * @param config_filename configuration file used by the webhook. @@ -1792,6 +1792,17 @@ TALER_TESTING_cmd_webhook (const char *label, const char *config_filename); +/** + * Command to run the 'taler-merchant-exchange' program. + * + * @param label command label. + * @param config_filename configuration file used by the webhook. + */ +struct TALER_TESTING_Command +TALER_TESTING_cmd_run_tme (const char *label, + const char *config_filename); + + /** * This function is used to start the web server. * diff --git a/src/include/taler_merchantdb_plugin.h b/src/include/taler_merchantdb_plugin.h index 30609bae..7a15ddee 100644 --- a/src/include/taler_merchantdb_plugin.h +++ b/src/include/taler_merchantdb_plugin.h @@ -1971,6 +1971,28 @@ struct TALER_MERCHANTDB_Plugin void *cb_cls); + /** + * Update transfer status. + * + * @param cls closure + * @param exchange_url the exchange that made the transfer + * @param wtid wire transfer subject + * @param next_attempt when should we try again (if ever) + * @param ec current error state of checking the transfer + * @param failed true if validation has failed for good + * @param verified true if validation has succeeded for good + * @return database transaction status + */ + enum GNUNET_DB_QueryStatus + (*update_transfer_status)( + void *cls, + const char *exchange_url, + const struct TALER_WireTransferIdentifierRawP *wtid, + struct GNUNET_TIME_Absolute next_attempt, + enum TALER_ErrorCode ec, + bool failed, + bool verified); + /** * Retrieve wire transfer details of wire details * that taler-merchant-exchange still needs to diff --git a/src/testing/Makefile.am b/src/testing/Makefile.am index e04b6c7f..527d63e5 100644 --- a/src/testing/Makefile.am +++ b/src/testing/Makefile.am @@ -72,6 +72,7 @@ libtalermerchanttesting_la_SOURCES = \ testing_api_cmd_refund_order.c \ testing_api_cmd_tip_authorize.c \ testing_api_cmd_tip_pickup.c \ + testing_api_cmd_tme.c \ testing_api_cmd_wallet_get_order.c \ testing_api_cmd_wallet_get_tip.c \ testing_api_cmd_wallet_post_orders_refund.c \ diff --git a/src/testing/test_kyc_api.c b/src/testing/test_kyc_api.c index 902a46ce..e000778f 100644 --- a/src/testing/test_kyc_api.c +++ b/src/testing/test_kyc_api.c @@ -258,6 +258,8 @@ run (void *cls, MHD_HTTP_OK, "deposit-simple", NULL), + TALER_TESTING_cmd_run_tme ("run taler-merchant-exchange-1", + CONFIG_FILE), TALER_TESTING_cmd_merchant_get_transfers ("get-transfers-1", merchant_url, merchant_payto, @@ -369,6 +371,8 @@ run (void *cls, MHD_HTTP_OK, "deposit-simple", NULL), + TALER_TESTING_cmd_run_tme ("run taler-merchant-exchange-2-aml", + CONFIG_FILE), TALER_TESTING_cmd_merchant_get_transfers ("get-transfers-aml", merchant_url, merchant_payto, diff --git a/src/testing/test_merchant_api.c b/src/testing/test_merchant_api.c index 99ed291b..551f824c 100644 --- a/src/testing/test_merchant_api.c +++ b/src/testing/test_merchant_api.c @@ -495,6 +495,10 @@ run (void *cls, MHD_HTTP_OK, "deposit-simple", NULL), + TALER_TESTING_cmd_run_tme ("run taler-merchant-exchange-1", + config_file), + /* FIXME: with the new API, the following will no longer + make sense (probably should just be removed): */ TALER_TESTING_cmd_merchant_post_transfer2 ("post-transfer-bad", merchant_url, PAYTO_I1, diff --git a/src/testing/testing_api_cmd_tme.c b/src/testing/testing_api_cmd_tme.c new file mode 100644 index 00000000..758083a5 --- /dev/null +++ b/src/testing/testing_api_cmd_tme.c @@ -0,0 +1,161 @@ +/* + This file is part of TALER + Copyright (C) 2023 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as + published by the Free Software Foundation; either version 3, + or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public + License along with TALER; see the file COPYING. If not, see + +*/ +/** + * @file testing/testing_api_cmd_tme.c + * @brief run the taler-merchant-exchange command + * @author Christian Grothoff + */ +#include "platform.h" +#include "taler/taler_json_lib.h" +#include +#include "taler/taler_signatures.h" +#include "taler/taler_testing_lib.h" + + +/** + * State for a "taler-merchant-exchange" CMD. + */ +struct MerchantExchangeState +{ + + /** + * Process for taler-merchant-exchange + */ + struct GNUNET_OS_Process *merchant_exchange_proc; + + /** + * Configuration file used by the program. + */ + const char *config_filename; +}; + + +/** + * Run the command; use the `taler-merchant-exchange' program. + * + * @param cls closure. + * @param cmd command currently being executed. + * @param is interpreter state. + */ +static void +tme_run (void *cls, + const struct TALER_TESTING_Command *cmd, + struct TALER_TESTING_Interpreter *is) +{ + struct MerchantExchangeState *ws = cls; + + (void) cmd; + ws->merchant_exchange_proc + = GNUNET_OS_start_process (GNUNET_OS_INHERIT_STD_ALL, + NULL, NULL, NULL, + "taler-merchant-exchange", + "taler-merchant-exchange", + "-c", ws->config_filename, + "-t", /* exit when done */ + "-L", "DEBUG", + NULL); + if (NULL == ws->merchant_exchange_proc) + { + GNUNET_break (0); + TALER_TESTING_interpreter_fail (is); + return; + } + TALER_TESTING_wait_for_sigchld (is); +} + + +/** + * Free the state of a "exchange" CMD, and possibly + * kills its process if it did not terminate regularly. + * + * @param cls closure. + * @param cmd the command being freed. + */ +static void +tme_cleanup (void *cls, + const struct TALER_TESTING_Command *cmd) +{ + struct MerchantExchangeState *ws = cls; + + (void) cmd; + if (NULL != ws->merchant_exchange_proc) + { + GNUNET_break (0 == + GNUNET_OS_process_kill (ws->merchant_exchange_proc, + SIGKILL)); + GNUNET_OS_process_wait (ws->merchant_exchange_proc); + GNUNET_OS_process_destroy (ws->merchant_exchange_proc); + ws->merchant_exchange_proc = NULL; + } + GNUNET_free (ws); +} + + +/** + * Offer "tme" CMD internal data to other commands. + * + * @param cls closure. + * @param[out] ret result. + * @param trait name of the trait. + * @param index index number of the object to offer. + * @return #GNUNET_OK on success. + */ +static enum GNUNET_GenericReturnValue +tme_traits (void *cls, + const void **ret, + const char *trait, + unsigned int index) +{ + struct MerchantExchangeState *ws = cls; + struct TALER_TESTING_Trait traits[] = { + TALER_TESTING_make_trait_process (&ws->merchant_exchange_proc), + TALER_TESTING_trait_end () + }; + + return TALER_TESTING_get_trait (traits, + ret, + trait, + index); +} + + +struct TALER_TESTING_Command +TALER_TESTING_cmd_run_tme (const char *label, + const char *config_filename) +{ + struct MerchantExchangeState *ws; + + ws = GNUNET_new (struct MerchantExchangeState); + ws->config_filename = config_filename; + + { + struct TALER_TESTING_Command cmd = { + .cls = ws, + .label = label, + .run = &tme_run, + .cleanup = &tme_cleanup, + .traits = &tme_traits + }; + + return cmd; + } +} + + +/* end of testing_api_cmd_tme.c */ -- cgit v1.2.3