towards idempotency in POST /private/transfers

1 files changed, 161 insertions, 116 deletions

diff --git a/src/backend/taler-merchant-httpd_private-post-transfers.c b/src/backend/taler-merchant-httpd_private-post-transfers.c
index fd9752d8..519e69f5 100644
--- a/src/backend/taler-merchant-httpd_private-post-transfers.c
+++ b/src/backend/taler-merchant-httpd_private-post-transfers.c
@@ -299,11 +299,12 @@ check_transfer (void *cls,
     GNUNET_break_op (0);
     ptc->check_transfer_result = GNUNET_SYSERR;
     /* Build the `TrackTransferConflictDetails` */
+    ptc->response_code = MHD_HTTP_ACCEPTED;
     ptc->response
       = TALER_MHD_make_json_pack (
           "{s:I, s:s, s:s, s:o, s:o,"
           " s:I, s:o, s:o, s:o, s:o,"
-          " s:o, s:o, s:o }",
+          " s:o, s:o, s:o, s:o, s:o }",
           "code",
           (json_int_t) TALER_EC_POST_TRANSFERS_CONFLICTING_REPORTS,
           "hint",
@@ -330,6 +331,10 @@ check_transfer (void *cls,
           GNUNET_JSON_from_data_auto (&ttd->h_contract_terms),
           "amount_with_fee",
           TALER_JSON_from_amount (amount_with_fee),
+          "coin_value",
+          TALER_JSON_from_amount (&ttd->coin_value),
+          "coin_fee",
+          TALER_JSON_from_amount (&ttd->coin_fee),
           "deposit_fee",
           TALER_JSON_from_amount (deposit_fee));
     return;
@@ -339,16 +344,14 @@ check_transfer (void *cls,
 
 
 /**
- * Check that the given @a wire_fee is what the
- * @a exchange_pub should charge at the @a execution_time.
- * If the fee is correct (according to our database),
- * return #GNUNET_OK.  If we do not have the fee structure
- * in our DB, we just accept it and return #GNUNET_NO;
- * if we have proof that the fee is bogus, we respond with
- * the proof to the client and return #GNUNET_SYSERR.
+ * Check that the given @a wire_fee is what the @a exchange_pub should charge
+ * at the @a execution_time.  If the fee is correct (according to our
+ * database), return #GNUNET_OK.  If we do not have the fee structure in our
+ * DB, we just accept it and return #GNUNET_NO; if we have proof that the fee
+ * is bogus, we respond with the proof to the client and return
+ * #GNUNET_SYSERR.
  *
  * @param ptc context of the transfer to respond to
- * @param json response from the exchange
  * @param execution_time time of the wire transfer
  * @param wire_fee fee claimed by the exchange
  * @return #GNUNET_SYSERR if we returned hard proof of
@@ -356,7 +359,6 @@ check_transfer (void *cls,
  */
 static int
 check_wire_fee (struct PostTransfersContext *ptc,
-                const json_t *json,
                 struct GNUNET_TIME_Absolute execution_time,
                 const struct TALER_Amount *wire_fee)
 {
@@ -397,11 +399,14 @@ check_wire_fee (struct PostTransfersContext *ptc,
     return GNUNET_OK;   /* expected_fee >= wire_fee */
   }
   /* Wire fee check failed, export proof to client */
-  resume_transfer_with_response (
-    ptc,
-    MHD_HTTP_FAILED_DEPENDENCY,
+  /* FIXME: This is not actually the *full* proof, as we are
+     not including the exchange's bogus response with the
+     signature claiming a different wire fee.  Also, this
+     error is not described in the API docs! */
+  ptc->response_code = MHD_HTTP_ACCEPTED;
+  ptc->response =
     TALER_MHD_make_json_pack (
-      "{s:I, s:o, s:o, s:o, s:o, s:o, s:o, s:o, s:o, s:O}",
+      "{s:I, s:o, s:o, s:o, s:o, s:o, s:o, s:o, s:o}",
       "code", (json_int_t) TALER_EC_POST_TRANSFERS_JSON_BAD_WIRE_FEE,
       "wire_fee", TALER_JSON_from_amount (wire_fee),
       "execution_time", GNUNET_JSON_from_time_abs (execution_time),
@@ -410,8 +415,7 @@ check_wire_fee (struct PostTransfersContext *ptc,
       "start_date", GNUNET_JSON_from_time_abs (start_date),
       "end_date", GNUNET_JSON_from_time_abs (end_date),
       "master_sig", GNUNET_JSON_from_data_auto (&master_sig),
-      "master_pub", GNUNET_JSON_from_data_auto (&ptc->master_pub),
-      "json", json));
+      "master_pub", GNUNET_JSON_from_data_auto (&ptc->master_pub));
   GNUNET_free (wire_method);
   return GNUNET_SYSERR;
 }
@@ -452,85 +456,6 @@ wire_transfer_cb (void *cls,
     return;
   }
 
-  if (GNUNET_SYSERR ==
-      check_wire_fee (ptc,
-                      hr->reply,
-                      td->execution_time,
-                      &td->wire_fee))
-    return;
-
-  /* Now we want to double-check that any (Taler coin) deposit
-   * which is accounted into _this_ wire transfer, does exist
-   * into _our_ database.  This is the rationale: if the
-   * exchange paid us for it, we must have received it _beforehands_!
-   *
-   * details_length is how many (Taler coin) deposits have been
-   * aggregated into _this_ wire transfer.
-   *///
-  for (unsigned int i = 0; i < td->details_length; i++)
-  {
-    const struct TALER_TrackTransferDetails *ttd = &td->details[i];
-
-    ptc->current_offset = i;
-    ptc->current_detail = ttd;
-    /* Set the coin as "never seen" before. */
-    ptc->check_transfer_result = GNUNET_NO;
-    TMH_db->preflight (TMH_db->cls);
-    qs = TMH_db->lookup_deposits_by_contract_and_coin (TMH_db->cls,
-                                                       instance_id,
-                                                       &ttd->h_contract_terms,
-                                                       &ttd->coin_pub,
-                                                       &check_transfer,
-                                                       ptc);
-    if (0 > qs)
-    {
-      /* single, read-only SQL statements should never cause
-         serialization problems */
-      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs);
-      /* Always report on hard error as well to enable diagnostics */
-      GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs);
-      resume_transfer_with_error (
-        ptc,
-        MHD_HTTP_INTERNAL_SERVER_ERROR,
-        TALER_EC_POST_TRANSFERS_DB_FETCH_DEPOSIT_ERROR,
-        "failed to obtain deposit data from local database");
-      return;
-    }
-    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
-    {
-      /* The exchange says we made this deposit, but WE do not
-         recall making it (corrupted / unreliable database?)!
-         Well, let's say thanks and accept the money! */
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "Failed to find payment data in DB\n");
-      ptc->check_transfer_result = GNUNET_OK;
-    }
-    if (GNUNET_NO == ptc->check_transfer_result)
-    {
-      /* Internal error: how can we have called #check_transfer()
-         but still have no result? */
-      GNUNET_break (0);
-      resume_transfer_with_error (ptc,
-                                  MHD_HTTP_INTERNAL_SERVER_ERROR,
-                                  TALER_EC_POST_TRANSFERS_DB_INTERNAL_LOGIC_ERROR,
-                                  "internal logic error");
-      return;
-    }
-    if (GNUNET_SYSERR == ptc->check_transfer_result)
-    {
-      /* #check_transfer() failed, report conflict! */
-      GNUNET_break_op (0);
-      GNUNET_assert (NULL != ptc->response);
-      resume_transfer_with_response (ptc,
-                                     MHD_HTTP_FAILED_DEPENDENCY,
-                                     ptc->response);
-      ptc->response = NULL;
-      return;
-    }
-  }
-
-  /* Response is consistent with the /deposit we made,
-     remember it for future reference */
   for (unsigned int r = 0; r<MAX_RETRIES; r++)
   {
     TMH_db->preflight (TMH_db->cls);
@@ -674,6 +599,81 @@ process_transfer_with_exchange (void *cls,
 
 
 /**
+ * Now we want to double-check that any (Taler coin) deposit which is
+ * accounted into _this_ wire transfer, does exist into _our_ database.  This
+ * is the rationale: if the exchange paid us for it, we must have received it
+ * _beforehands_!
+ *
+ * @param cls a `struct PostTransfersContext`
+ * @param current_offset at which offset in the exchange's reply are the @a ttd
+ * @param ttd details about an aggregated transfer (to check)
+ */
+static void
+verify_exchange_claim_cb (void *cls,
+                          unsigned int current_offset,
+                          const struct TALER_TrackTransferDetails *ttd)
+{
+  struct PostTransfersContext *ptc = cls;
+  enum GNUNET_DB_QueryStatus qs;
+
+  if (0 != ptc->response_code)
+    return; /* already encountered an error */
+  ptc->current_offset = current_offset;
+  ptc->current_detail = ttd;
+  /* Set the coin as "never seen" before. */
+  ptc->check_transfer_result = GNUNET_NO;
+  TMH_db->preflight (TMH_db->cls);
+  qs = TMH_db->lookup_deposits_by_contract_and_coin (
+    TMH_db->cls,
+    ptc->hc->instance->settings.id,
+    &ttd->h_contract_terms,
+    &ttd->coin_pub,
+    &check_transfer,
+    ptc);
+  if (0 > qs)
+  {
+    /* single, read-only SQL statements should never cause
+       serialization problems */
+    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs);
+    /* Always report on hard error as well to enable diagnostics */
+    GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs);
+    ptc->response_code = MHD_HTTP_INTERNAL_SERVER_ERROR;
+    ptc->response
+      = TALER_MHD_make_error (TALER_EC_POST_TRANSFERS_DB_FETCH_DEPOSIT_ERROR,
+                              "failed to obtain deposit data from local database");
+    return;
+  }
+  if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+  {
+    /* The exchange says we made this deposit, but WE do not
+       recall making it (corrupted / unreliable database?)!
+       Well, let's say thanks and accept the money! */
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                "Failed to find payment data in DB\n");
+    ptc->check_transfer_result = GNUNET_OK;
+  }
+  if (GNUNET_NO == ptc->check_transfer_result)
+  {
+    /* Internal error: how can we have called #check_transfer()
+       but still have no result? */
+    GNUNET_break (0);
+    ptc->response_code = MHD_HTTP_INTERNAL_SERVER_ERROR;
+    ptc->response =
+      TALER_MHD_make_error (TALER_EC_POST_TRANSFERS_DB_INTERNAL_LOGIC_ERROR,
+                            "internal logic error");
+    return;
+  }
+  if (GNUNET_SYSERR == ptc->check_transfer_result)
+  {
+    /* #check_transfer() failed, report conflict! */
+    GNUNET_break_op (0);
+    GNUNET_assert (NULL != ptc->response);
+    return;
+  }
+}
+
+
+/**
  * Represents an entry in the table used to sum up
  * individual deposits for each h_contract_terms/order_id
  * (as the exchange gives us per coin, and we return
@@ -710,7 +710,7 @@ struct Entry
  * @param deposit_fee the fee charged for @a deposit_value
  */
 static void
-transfer_details_cb (void *cls,
+transfer_summary_cb (void *cls,
                      const char *order_id,
                      const struct TALER_Amount *deposit_value,
                      const struct TALER_Amount *deposit_fee)
@@ -846,6 +846,7 @@ TMH_private_post_transfers (const struct TMH_RequestHandler *rh,
     hc->cc = &transfer_cleanup;
   }
 
+queue:
   if (0 != ptc->response_code)
   {
     MHD_RESULT ret;
@@ -910,43 +911,86 @@ TMH_private_post_transfers (const struct TMH_RequestHandler *rh,
 
   /* Check if transfer data is in database! */
   {
-    struct GNUNET_CONTAINER_MultiHashMap *map;
     struct GNUNET_TIME_Absolute execution_time;
     struct TALER_Amount total_amount;
     struct TALER_Amount wire_fee;
+    bool verified;
 
     TMH_db->preflight (TMH_db->cls);
-    map = GNUNET_CONTAINER_multihashmap_create (16,
-                                                GNUNET_NO);
-    qs = TMH_db->lookup_transfer_details (TMH_db->cls,
-                                          hc->instance->settings.id,
-                                          ptc->exchange_url,
-                                          ptc->payto_uri,
-                                          &ptc->wtid,
-                                          &total_amount,
-                                          &wire_fee,
-                                          execution_time,
-                                          &transfer_details_cb,
-                                          map);
+    qs = TMH_db->lookup_transfer (TMH_db->cls,
+                                  ptc->exchange_url,
+                                  &ptc->wtid,
+                                  &total_amount,
+                                  &wire_fee,
+                                  &execution_time,
+                                  &verified);
     if (0 > qs)
     {
       /* Simple select queries should not cause serialization issues */
       GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs);
       /* Always report on hard error as well to enable diagnostics */
       GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs);
-      GNUNET_CONTAINER_multihashmap_iterate (map,
-                                             &hashmap_free,
-                                             NULL);
-      GNUNET_CONTAINER_multihashmap_destroy (map);
       return TALER_MHD_reply_with_error (connection,
                                          MHD_HTTP_INTERNAL_SERVER_ERROR,
                                          TALER_EC_POST_TRANSFERS_DB_LOOKUP_ERROR,
                                          "Failed to query database about transfer details");
     }
-    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != qs)
+    if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+      goto fetch;
+    if (! verified)
+    {
+      if (GNUNET_SYSERR ==
+          check_wire_fee (ptc,
+                          execution_time,
+                          &wire_fee))
+      {
+        GNUNET_assert (0 != ptc->response_code);
+        goto queue;
+      }
+
+      qs = TMH_db->lookup_transfer_details (TMH_db->cls,
+                                            ptc->exchange_url,
+                                            &ptc->wtid,
+                                            &verify_exchange_claim_cb,
+                                            ptc);
+      if (0 != ptc->response_code)
+        goto queue;
+      verified = true;
+      qs = TMH_db->set_transfer_status_to_verified (TMH_db->cls,
+                                                    ptc->exchange_url,
+                                                    &ptc->wtid);
+      GNUNET_break (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs);
+    }
+
+    /* Short version: we already verified, generate the summary response */
+    GNUNET_assert (verified);
     {
+      struct GNUNET_CONTAINER_MultiHashMap *map;
       json_t *deposit_sums;
 
+      map = GNUNET_CONTAINER_multihashmap_create (16,
+                                                  GNUNET_NO);
+      qs = TMH_db->lookup_transfer_summary (TMH_db->cls,
+                                            ptc->exchange_url,
+                                            &ptc->wtid,
+                                            &transfer_summary_cb,
+                                            map);
+      if (0 > qs)
+      {
+        /* Simple select queries should not cause serialization issues */
+        GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR != qs);
+        /* Always report on hard error as well to enable diagnostics */
+        GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs);
+        GNUNET_CONTAINER_multihashmap_iterate (map,
+                                               &hashmap_free,
+                                               NULL);
+        GNUNET_CONTAINER_multihashmap_destroy (map);
+        return TALER_MHD_reply_with_error (connection,
+                                           MHD_HTTP_INTERNAL_SERVER_ERROR,
+                                           TALER_EC_POST_TRANSFERS_DB_LOOKUP_ERROR,
+                                           "Failed to query database about transfer details");
+      }
+
       deposit_sums = json_array ();
       GNUNET_assert (NULL != deposit_sums);
       GNUNET_CONTAINER_multihashmap_iterate (map,
@@ -961,11 +1005,12 @@ TMH_private_post_transfers (const struct TMH_RequestHandler *rh,
         "wire_fee", TALER_JSON_from_amount (&wire_fee),
         "execution_time", GNUNET_JSON_from_time_abs (execution_time),
         "deposit_sums", deposit_sums);
-    }
-  }
+    } /* end of 'verified == true' */
+  } /* end of 'transfer data in database' */
 
   /* reply not in database, ensure the POST is in the database, and
      start work to obtain the reply from the exchange */
+fetch:
   qs = TMH_db->insert_transfer (TMH_db->cls,
                                 ptc->hc->instance->settings.id,
                                 ptc->exchange_url,