From 69ed15654bc591d118e87e6acfe4d8931097d5fa Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sun, 9 May 2021 11:10:16 +0200 Subject: make POST /private/transfers a proper transaction (relates to #6854, but unclear if it fixes anything) --- .../taler-merchant-httpd_private-post-transfers.c | 491 +++++++++++++-------- 1 file changed, 298 insertions(+), 193 deletions(-) (limited to 'src') diff --git a/src/backend/taler-merchant-httpd_private-post-transfers.c b/src/backend/taler-merchant-httpd_private-post-transfers.c index 80998e85..9a425e9c 100644 --- a/src/backend/taler-merchant-httpd_private-post-transfers.c +++ b/src/backend/taler-merchant-httpd_private-post-transfers.c @@ -1,6 +1,6 @@ /* This file is part of TALER - (C) 2014-2020 Taler Systems SA + (C) 2014-2021 Taler Systems SA TALER is free software; you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software @@ -135,6 +135,11 @@ struct PostTransfersContext * #GNUNET_OK if we did find a matching coin. */ int check_transfer_result; + + /** + * Should we retry the transaction due to a serialization error? + */ + bool soft_retry; }; @@ -149,9 +154,6 @@ static struct PostTransfersContext *ptc_head; static struct PostTransfersContext *ptc_tail; -/** - * We are shutting down, force resume of all POST /transfers requests. - */ void TMH_force_post_transfers_resume () { @@ -250,6 +252,11 @@ transfer_cleanup (void *cls) TALER_EXCHANGE_transfers_get_cancel (ptc->wdh); ptc->wdh = NULL; } + if (NULL != ptc->response) + { + MHD_destroy_response (ptc->response); + ptc->response = NULL; + } GNUNET_free (ptc); } @@ -595,6 +602,8 @@ verify_exchange_claim_cb (void *cls, if (0 != ptc->response_code) return; /* already encountered an error */ + if (ptc->soft_retry) + return; /* already encountered an error */ ptc->current_offset = current_offset; ptc->current_detail = ttd; /* Set the coin as "never seen" before. */ @@ -607,27 +616,28 @@ verify_exchange_claim_cb (void *cls, &ttd->coin_pub, &check_transfer, ptc); - if (0 > qs) + switch (qs) { - /* single, read-only SQL statements should never cause - serialization problems */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs); - /* Always report on hard error as well to enable diagnostics */ - GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); + case GNUNET_DB_STATUS_SOFT_ERROR: + ptc->soft_retry = true; + return; + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); ptc->response_code = MHD_HTTP_INTERNAL_SERVER_ERROR; ptc->response = TALER_MHD_make_error (TALER_EC_GENERIC_DB_FETCH_FAILED, "deposit by contract and coin"); return; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: /* The exchange says we made this deposit, but WE do not recall making it (corrupted / unreliable database?)! Well, let's say thanks and accept the money! */ GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to find payment data in DB\n"); ptc->check_transfer_result = GNUNET_OK; + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; } if (GNUNET_NO == ptc->check_transfer_result) { @@ -799,15 +809,32 @@ handle_transfer_timeout (void *cls) /** - * Manages a POST /private/transfers call. It calls the GET /transfers/$WTID - * offered by the exchange in order to obtain the set of transfers - * (of coins) associated with a given wire transfer. + * We are *done* processing the request, just queue the response (!) * - * @param rh context of the handler - * @param connection the MHD connection to handle - * @param[in,out] hc context with further information about the request - * @return MHD result code + * @param ptc request context */ +static MHD_RESULT +queue (struct PostTransfersContext *ptc) +{ + MHD_RESULT ret; + + GNUNET_assert (0 != ptc->response_code); + if (UINT_MAX == ptc->response_code) + { + GNUNET_break (0); + return MHD_NO; /* hard error */ + } + ret = MHD_queue_response (ptc->connection, + ptc->response_code, + ptc->response); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Queueing response (%u) for POST /private/transfers (%s).\n", + (unsigned int) ptc->response_code, + ret ? "OK" : "FAILED"); + return ret; +} + + MHD_RESULT TMH_private_post_transfers (const struct TMH_RequestHandler *rh, struct MHD_Connection *connection, @@ -824,33 +851,8 @@ TMH_private_post_transfers (const struct TMH_RequestHandler *rh, hc->ctx = ptc; hc->cc = &transfer_cleanup; } - -queue: if (0 != ptc->response_code) - { - MHD_RESULT ret; - - /* We are *done* processing the request, just queue the response (!) */ - if (UINT_MAX == ptc->response_code) - { - GNUNET_break (0); - return MHD_NO; /* hard error */ - } - ret = MHD_queue_response (connection, - ptc->response_code, - ptc->response); - if (NULL != ptc->response) - { - MHD_destroy_response (ptc->response); - ptc->response = NULL; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Queueing response (%u) for POST /private/transfers (%s).\n", - (unsigned int) ptc->response_code, - ret ? "OK" : "FAILED"); - return ret; - } - + return queue (ptc); if ( (NULL != ptc->fo) || (NULL != ptc->wdh) ) { @@ -859,7 +861,6 @@ queue: "Not sure why we are here, should be suspended\n"); return MHD_YES; /* still work in progress */ } - if (NULL == ptc->exchange_url) { /* First request, parse it! */ @@ -874,21 +875,19 @@ queue: &ptc->exchange_url), GNUNET_JSON_spec_end () }; - - { - enum GNUNET_GenericReturnValue res; - - res = TALER_MHD_parse_json_data (connection, - hc->request_body, - spec); - if (GNUNET_OK != res) - return (GNUNET_NO == res) - ? MHD_YES - : MHD_NO; - } + enum GNUNET_GenericReturnValue res; + + res = TALER_MHD_parse_json_data (connection, + hc->request_body, + spec); + if (GNUNET_OK != res) + return (GNUNET_NO == res) + ? MHD_YES + : MHD_NO; } /* Check if transfer data is in database! */ + for (unsigned int retry = 0; retrypreflight (TMH_db->cls); + if (GNUNET_OK != + TMH_db->start (TMH_db->cls, + "post-transfers")) + { + GNUNET_break (0); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_START_FAILED, + "transfer"); + } qs = TMH_db->lookup_transfer (TMH_db->cls, ptc->exchange_url, &ptc->wtid, @@ -903,151 +912,247 @@ queue: &wire_fee, &execution_time, &verified); - if (0 > qs) + switch (qs) { - /* Simple select queries should not cause serialization issues */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs); - /* Always report on hard error as well to enable diagnostics */ - GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); return TALER_MHD_reply_with_error (connection, MHD_HTTP_INTERNAL_SERVER_ERROR, TALER_EC_GENERIC_DB_FETCH_FAILED, "transfer"); - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - goto fetch; - if (! verified) - { - if (GNUNET_SYSERR == - check_wire_fee (ptc, - execution_time, - &wire_fee)) + break; + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + /* Transfer unknown */ { - GNUNET_assert (0 != ptc->response_code); - goto queue; - } - - qs = TMH_db->lookup_transfer_details (TMH_db->cls, - ptc->exchange_url, - &ptc->wtid, - &verify_exchange_claim_cb, - ptc); - if (0 != ptc->response_code) - goto queue; - verified = true; - qs = TMH_db->set_transfer_status_to_verified (TMH_db->cls, - ptc->exchange_url, - &ptc->wtid); - GNUNET_break (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs); - } + uint64_t account_serial; + + /* Either the record already exists (we should ignore this), or + the INSERT failed because we did not find the account based on + the given payto-URI and the instance. */ + qs = TMH_db->lookup_account (TMH_db->cls, + ptc->hc->instance->settings.id, + ptc->payto_uri, + &account_serial); + switch (qs) + { + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "lookup_account"); + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Bank account `%s' not configured for instance `%s'\n", + ptc->payto_uri, + ptc->hc->instance->settings.id); + TMH_db->rollback (TMH_db->cls); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_NOT_FOUND, + TALER_EC_MERCHANT_PRIVATE_POST_TRANSFERS_ACCOUNT_NOT_FOUND, + ptc->payto_uri); + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + + qs = TMH_db->insert_transfer (TMH_db->cls, + ptc->hc->instance->settings.id, + ptc->exchange_url, + &ptc->wtid, + &ptc->amount, + ptc->payto_uri, + true /* confirmed! */); + switch (qs) + { + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_STORE_FAILED, + "transfer"); + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_assert (0); + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + + qs = TMH_db->commit (TMH_db->cls); + switch (qs) + { + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_COMMIT_FAILED, + NULL); + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Suspending POST /private/transfers handling while working with exchange\n"); + MHD_suspend_connection (connection); + GNUNET_CONTAINER_DLL_insert (ptc_head, + ptc_tail, + ptc); + ptc->fo = TMH_EXCHANGES_find_exchange (ptc->exchange_url, + NULL, + GNUNET_NO, + &process_transfer_with_exchange, + ptc); + ptc->timeout_task + = GNUNET_SCHEDULER_add_delayed (TRANSFER_GENERIC_TIMEOUT, + &handle_transfer_timeout, + ptc); + return MHD_YES; - /* Short version: we already verified, generate the summary response */ - GNUNET_assert (verified); - { - struct GNUNET_CONTAINER_MultiHashMap *map; - json_t *deposit_sums; - - map = GNUNET_CONTAINER_multihashmap_create (16, - GNUNET_NO); - qs = TMH_db->lookup_transfer_summary (TMH_db->cls, - ptc->exchange_url, - &ptc->wtid, - &transfer_summary_cb, - map); - if (0 > qs) + } + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + /* Transfer exists */ + if (! verified) + { + if (GNUNET_SYSERR == + check_wire_fee (ptc, + execution_time, + &wire_fee)) + { + TMH_db->rollback (TMH_db->cls); + return queue (ptc); /* generate error */ + } + qs = TMH_db->lookup_transfer_details (TMH_db->cls, + ptc->exchange_url, + &ptc->wtid, + &verify_exchange_claim_cb, + ptc); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "lookup_transfer_details"); + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + default: + break; + } + if (0 != ptc->response_code) + { + TMH_db->rollback (TMH_db->cls); + return queue (ptc); /* generate error */ + } + if (ptc->soft_retry) + { + /* DB serialization failure */ + ptc->soft_retry = false; + TMH_db->rollback (TMH_db->cls); + continue; + } + verified = true; + qs = TMH_db->set_transfer_status_to_verified (TMH_db->cls, + ptc->exchange_url, + &ptc->wtid); + switch (qs) + { + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "set_transfer_status_to_verified"); + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + GNUNET_assert (0); + break; + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + break; + } + } /* end of 'if (! verified)' */ + + /* Short version: we already verified, generate the summary response */ + GNUNET_assert (verified); { - /* Simple select queries should not cause serialization issues */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs); - /* Always report on hard error as well to enable diagnostics */ - GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); + struct GNUNET_CONTAINER_MultiHashMap *map; + json_t *deposit_sums; + + map = GNUNET_CONTAINER_multihashmap_create (16, + GNUNET_NO); + qs = TMH_db->lookup_transfer_summary (TMH_db->cls, + ptc->exchange_url, + &ptc->wtid, + &transfer_summary_cb, + map); + switch (qs) + { + case GNUNET_DB_STATUS_SOFT_ERROR: + TMH_db->rollback (TMH_db->cls); + continue; + case GNUNET_DB_STATUS_HARD_ERROR: + GNUNET_break (0); + TMH_db->rollback (TMH_db->cls); + GNUNET_CONTAINER_multihashmap_iterate (map, + &hashmap_free, + NULL); + GNUNET_CONTAINER_multihashmap_destroy (map); + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_FETCH_FAILED, + "transfer summary"); + case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: + case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: + default: + break; + } + + + deposit_sums = json_array (); + GNUNET_assert (NULL != deposit_sums); GNUNET_CONTAINER_multihashmap_iterate (map, &hashmap_free, - NULL); + deposit_sums); GNUNET_CONTAINER_multihashmap_destroy (map); - return TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_FETCH_FAILED, - "transfer summary"); - } - - deposit_sums = json_array (); - GNUNET_assert (NULL != deposit_sums); - GNUNET_CONTAINER_multihashmap_iterate (map, - &hashmap_free, - deposit_sums); - GNUNET_CONTAINER_multihashmap_destroy (map); - return TALER_MHD_reply_json_pack ( - connection, - MHD_HTTP_OK, - "{s:o,s:o,s:o,s:o}", - "total", TALER_JSON_from_amount (&total_amount), - "wire_fee", TALER_JSON_from_amount (&wire_fee), - "execution_time", GNUNET_JSON_from_time_abs (execution_time), - "deposit_sums", deposit_sums); - } /* end of 'verified == true' */ - } /* end of 'transfer data in database' */ - - /* reply not in database, ensure the POST is in the database, and - start work to obtain the reply from the exchange */ -fetch: - qs = TMH_db->insert_transfer (TMH_db->cls, - ptc->hc->instance->settings.id, - ptc->exchange_url, - &ptc->wtid, - &ptc->amount, - ptc->payto_uri, - true /* confirmed! */); - if (0 > qs) - { - /* Simple select queries should not cause serialization issues */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs); - /* Always report on hard error as well to enable diagnostics */ - GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); - return TALER_MHD_reply_with_error (connection, - MHD_HTTP_INTERNAL_SERVER_ERROR, - TALER_EC_GENERIC_DB_STORE_FAILED, - "transfer"); - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - uint64_t account_serial; - - /* Either the record already exists (we should ignore this), or - the INSERT failed because we did not find the account based on - the given payto-URI and the instance. */ - qs = TMH_db->lookup_account (TMH_db->cls, - ptc->hc->instance->settings.id, - ptc->payto_uri, - &account_serial); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Bank account `%s' not configured for instance `%s'\n", - ptc->payto_uri, - ptc->hc->instance->settings.id); - return TALER_MHD_reply_with_error (connection, - MHD_HTTP_NOT_FOUND, - TALER_EC_MERCHANT_PRIVATE_POST_TRANSFERS_ACCOUNT_NOT_FOUND, - ptc->payto_uri); - } - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Suspending POST /private/transfers handling while working with exchange\n"); - MHD_suspend_connection (connection); - GNUNET_CONTAINER_DLL_insert (ptc_head, - ptc_tail, - ptc); - ptc->fo = TMH_EXCHANGES_find_exchange (ptc->exchange_url, - NULL, - GNUNET_NO, - &process_transfer_with_exchange, - ptc); - ptc->timeout_task - = GNUNET_SCHEDULER_add_delayed (TRANSFER_GENERIC_TIMEOUT, - &handle_transfer_timeout, - ptc); - return MHD_YES; + return TALER_MHD_reply_json_pack ( + connection, + MHD_HTTP_OK, + "{s:o,s:o,s:o,s:o}", + "total", TALER_JSON_from_amount (&total_amount), + "wire_fee", TALER_JSON_from_amount (&wire_fee), + "execution_time", GNUNET_JSON_from_time_abs (execution_time), + "deposit_sums", deposit_sums); + } /* end of 'verified == true' (not an 'if'!) */ + } /* end of 'switch (qs)' */ + } /* end of 'for(retries...) */ + return TALER_MHD_reply_with_error (connection, + MHD_HTTP_INTERNAL_SERVER_ERROR, + TALER_EC_GENERIC_DB_SOFT_FAILURE, + "post-transfers"); } -- cgit v1.2.3