path: root/src/backend/taler-merchant-httpd_exchanges.c

/*
  This file is part of TALER
  (C) 2014-2023 Taler Systems SA

  TALER is free software; you can redistribute it and/or modify it under the
  terms of the GNU Affero General Public License as published by the Free Software
  Foundation; either version 3, or (at your option) any later version.

  TALER is distributed in the hope that it will be useful, but WITHOUT ANY
  WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
  A PARTICULAR PURPOSE.  See the GNU General Public License for more details.

  You should have received a copy of the GNU General Public License along with
  TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
*/
/**
 * @file taler-merchant-httpd_exchanges.c
 * @brief logic this HTTPD keeps for each exchange we interact with
 * @author Marcello Stanisci
 * @author Christian Grothoff
 */
#include "platform.h"
#include <taler/taler_json_lib.h>
#include <taler/taler_dbevents.h>
#include "taler-merchant-httpd_exchanges.h"
#include "taler-merchant-httpd.h"
#include <regex.h>

/**
 * How often do we retry DB transactions with soft errors?
 */
#define MAX_RETRIES 3

/**
 * Threshold after which exponential backoff should not increase.
 */
#define RETRY_BACKOFF_THRESHOLD GNUNET_TIME_relative_multiply ( \
    GNUNET_TIME_UNIT_SECONDS, 60)

/**
 * This is how long /keys long-polls for, so we should
 * allow this time between requests if there is no
 * answer. See exchange_api_handle.c.
 */
#define LONG_POLL_THRESHOLD GNUNET_TIME_relative_multiply ( \
    GNUNET_TIME_UNIT_SECONDS, 120)


/**
 * Information we keep for a pending #MMH_EXCHANGES_keys4exchange() operation.
 */
struct TMH_EXCHANGES_KeysOperation
{

  /**
   * Kept in a DLL.
   */
  struct TMH_EXCHANGES_KeysOperation *next;

  /**
   * Kept in a DLL.
   */
  struct TMH_EXCHANGES_KeysOperation *prev;

  /**
   * Function to call with the result.
   */
  TMH_EXCHANGES_Find2Continuation fc;

  /**
   * Closure for @e fc.
   */
  void *fc_cls;

  /**
   * Exchange we wait for the /keys for.
   */
  struct TMH_Exchange *my_exchange;

  /**
   * Task scheduled to asynchronously return the result to
   * the find continuation.
   */
  struct GNUNET_SCHEDULER_Task *at;

};


/**
 * Information about wire transfer fees of an exchange, by wire method.
 */
struct FeesByWireMethod
{

  /**
   * Kept in a DLL.
   */
  struct FeesByWireMethod *next;

  /**
   * Kept in a DLL.
   */
  struct FeesByWireMethod *prev;

  /**
   * Wire method these fees are for.
   */
  char *wire_method;

  /**
   * Applicable fees, NULL if unknown/error.
   */
  struct TALER_EXCHANGE_WireAggregateFees *af;

};


/**
 * Restriction that applies to an exchange account.
 */
struct Restriction
{
  /**
   * Kept in a DLL.
   */
  struct Restriction *next;

  /**
   * Kept in a DLL.
   */
  struct Restriction *prev;

  /**
   * Type of restriction imposed on the account.
   */
  enum TALER_EXCHANGE_AccountRestrictionType type;

  /**
   * Details depending on @e type.
   */
  union
  {

    /**
     * Accounts must match the given regular expression.
     */
    struct
    {

      /**
       * Pre-compiled regex.
       */
      regex_t ex;

    } regex;

  } details;
};


/**
 * Information about a bank account of the exchange.
 */
struct ExchangeAccount
{
  /**
   * Kept in a DLL.
   */
  struct ExchangeAccount *next;

  /**
   * Kept in a DLL.
   */
  struct ExchangeAccount *prev;

  /**
   * Wire method of this exchange account.
   */
  char *wire_method;

  /**
   * Currency conversion that applies to this account,
   * NULL if none.
   */
  char *conversion_url;

  /**
   * Head of DLL of debit restrictions of this account.
   */
  struct Restriction *d_head;

  /**
   * Tail of DLL of debit restrictions of this account.
   */
  struct Restriction *d_tail;
};


/**
 * Internal representation for an exchange
 */
struct TMH_Exchange
{

  /**
   * Kept in a DLL.
   */
  struct TMH_Exchange *next;

  /**
   * Kept in a DLL.
   */
  struct TMH_Exchange *prev;

  /**
   * Head of FOs pending for this exchange.
   */
  struct TMH_EXCHANGES_KeysOperation *keys_head;

  /**
   * Tail of FOs pending for this exchange.
   */
  struct TMH_EXCHANGES_KeysOperation *keys_tail;

  /**
   * Head of accounts of this exchange.
   */
  struct ExchangeAccount *acc_head;

  /**
   * Tail of accounts of this exchange.
   */
  struct ExchangeAccount *acc_tail;

  /**
   * (base) URL of the exchange.
   */
  char *url;

  /**
   * Currency offered by the exchange according to OUR configuration.
   */
  char *currency;

  /**
   * A connection to this exchange
   */
  struct TALER_EXCHANGE_GetKeysHandle *conn;

  /**
   * The keys of this exchange
   */
  struct TALER_EXCHANGE_Keys *keys;

  /**
   * Head of wire fees from /wire request.
   */
  struct FeesByWireMethod *wire_fees_head;

  /**
   * Tail of wire fees from /wire request.
   */
  struct FeesByWireMethod *wire_fees_tail;

  /**
   * Master public key of the exchange.
   */
  struct TALER_MasterPublicKeyP master_pub;

  /**
   * How soon can may we, at the earliest, re-download /keys?
   */
  struct GNUNET_TIME_Absolute first_retry;

  /**
   * How long should we wait between the next retry?
   */
  struct GNUNET_TIME_Relative retry_delay;

  /**
   * How long should we wait between the next retry for /wire?
   */
  struct GNUNET_TIME_Relative wire_retry_delay;

  /**
   * Task where we retry fetching /keys from the exchange.
   */
  struct GNUNET_SCHEDULER_Task *retry_task;

  /**
   * What state is this exchange in?
   */
  enum
  {

    /**
     * Downloading /keys failed.
     */
    ESTATE_FAILED = -1,

    /**
     * Nothing was ever done.
     */
    ESTATE_INIT = 0,

    /**
     * We are actively downloading /keys for the first time.
     */
    ESTATE_DOWNLOADING_FIRST = 1,

    /**
     * We finished downloading /keys and the exchange is
     * ready.
     */
    ESTATE_DOWNLOADED = 2,

    /**
     * We are downloading /keys again after a previous
     * success.
     */
    ESTATE_REDOWNLOADING_SUCCESS = 3,

    /**
     * We are downloading /keys again after a previous
     * failure.
     */
    ESTATE_REDOWNLOADING_FAILURE = 4

  } state;

  /**
   * true if this exchange is from our configuration and
   * explicitly trusted, false if we need to check each
   * key to be sure it is trusted.
   */
  bool trusted;

};


/**
 * Head of exchanges we know about.
 */
static struct TMH_Exchange *exchange_head;

/**
 * Tail of exchanges we know about.
 */
static struct TMH_Exchange *exchange_tail;

/**
 * Our event handler listening for /keys downloads
 * being put into the database.
 */
static struct GNUNET_DB_EventHandler *keys_eh;

/**
 * How many exchanges do we trust (for our configured
 * currency) as per our configuration? Used for a
 * sanity-check on startup.
 */
static int trusted_exchange_count;


const struct TALER_MasterPublicKeyP *
TMH_EXCHANGES_get_master_pub (
  const struct TMH_Exchange *exchange)
{
  GNUNET_break ( (exchange->trusted) ||
                 (NULL != exchange->keys) );
  return &exchange->master_pub;
}


const char *
TMH_EXCHANGES_get_currency (
  const struct TMH_Exchange *exchange)
{
  return exchange->currency;
}


/**
 * Free data structures within @a ea, but not @a ea
 * pointer itself.
 *
 * @param[in] ea data structure to free
 */
static void
free_exchange_account (struct ExchangeAccount *ea)
{
  struct Restriction *r;

  while (NULL != (r = ea->d_head))
  {
    GNUNET_CONTAINER_DLL_remove (ea->d_head,
                                 ea->d_tail,
                                 r);
    switch (r->type)
    {
    case TALER_EXCHANGE_AR_INVALID:
      GNUNET_assert (0);
      break;
    case TALER_EXCHANGE_AR_DENY:
      break;
    case TALER_EXCHANGE_AR_REGEX:
      regfree (&r->details.regex.ex);
      break;
    }
    GNUNET_free (r);
  }
  GNUNET_free (ea->wire_method);
  GNUNET_free (ea->conversion_url);
}


/**
 * Free list of all accounts in @a exchange.
 *
 * @param[in,out] exchange entry to free accounts for
 */
static void
purge_exchange_accounts (struct TMH_Exchange *exchange)
{
  struct ExchangeAccount *acc;

  while (NULL != (acc = exchange->acc_head))
  {
    GNUNET_CONTAINER_DLL_remove (exchange->acc_head,
                                 exchange->acc_tail,
                                 acc);
    free_exchange_account (acc);
    GNUNET_free (acc);
  }
}


/**
 * Lookup exchange by @a exchange_url. Create one
 * if it does not exist.
 *
 * @param exchange_url base URL to match against
 * @return fresh entry if exchange was not yet known
 */
static struct TMH_Exchange *
lookup_exchange (const char *exchange_url)
{
  struct TMH_Exchange *exchange;
  enum GNUNET_DB_QueryStatus qs;

  for (exchange = exchange_head;
       NULL != exchange;
       exchange = exchange->next)
    if (0 == strcmp (exchange->url,
                     exchange_url))
      return exchange;
  exchange = GNUNET_new (struct TMH_Exchange);
  exchange->url = GNUNET_strdup (exchange_url);
  GNUNET_CONTAINER_DLL_insert (exchange_head,
                               exchange_tail,
                               exchange);
  qs = TMH_db->select_exchange_keys (TMH_db->cls,
                                     exchange->url,
                                     &exchange->keys);
  GNUNET_break (qs >= 0);
  if (qs > 0)
    exchange->state = ESTATE_DOWNLOADED;
  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
              "The exchange `%s' is new (%d)\n",
              exchange_url,
              exchange->state);
  return exchange;
}


/**
 * Set the list of accounts of @a exchange.
 *
 * @param[in,out] exchange exchange to initialize or update
 * @param accounts_len length of @a accounts array
 * @param accounts array of accounts to convert
 * @return #GNUNET_OK on success
 */
static enum GNUNET_GenericReturnValue
set_exchange_accounts (
  struct TMH_Exchange *exchange,
  unsigned int accounts_len,
  const struct TALER_EXCHANGE_WireAccount accounts[static accounts_len])
{
  enum GNUNET_GenericReturnValue ret = GNUNET_OK;

  purge_exchange_accounts (exchange);
  for (unsigned int i = 0; i<accounts_len; i++)
  {
    const struct TALER_EXCHANGE_WireAccount *account = &accounts[i];
    struct ExchangeAccount *acc;

    acc = GNUNET_new (struct ExchangeAccount);
    acc->wire_method = TALER_payto_get_method (account->payto_uri);
    if (NULL != account->conversion_url)
      acc->conversion_url = GNUNET_strdup (account->conversion_url);
    GNUNET_CONTAINER_DLL_insert (exchange->acc_head,
                                 exchange->acc_tail,
                                 acc);
    for (unsigned int j = 0; j<account->debit_restrictions_length; j++)
    {
      const struct TALER_EXCHANGE_AccountRestriction *ar =
        &account->debit_restrictions[j];
      struct Restriction *r;

      r = GNUNET_new (struct Restriction);
      r->type = ar->type;
      switch (ar->type)
      {
      case TALER_EXCHANGE_AR_INVALID:
        GNUNET_assert (0);
        break;
      case TALER_EXCHANGE_AR_DENY:
        break;
      case TALER_EXCHANGE_AR_REGEX:
        if (0 != regcomp (&r->details.regex.ex,
                          ar->details.regex.posix_egrep,
                          REG_NOSUB | REG_EXTENDED))
        {
          GNUNET_break_op (0);
          GNUNET_free (r);
          ret = GNUNET_SYSERR;
          continue;
        }
        break;
      }
      GNUNET_CONTAINER_DLL_insert (acc->d_head,
                                   acc->d_tail,
                                   r);
    }
  }
  return ret;
}


/**
 * Function called with information about who is auditing
 * a particular exchange and what key the exchange is using.
 *
 * @param cls closure, will be `struct TMH_Exchange`
 * @param kr response details
 * @param[in] keys keys object returned
 */
static void
keys_mgmt_cb (
  void *cls,
  const struct TALER_EXCHANGE_KeysResponse *kr,
  struct TALER_EXCHANGE_Keys *keys);


/**
 * Check if we have any remaining pending requests for the
 * given @a exchange, and if we have the required data, call
 * the callback.
 *
 * @param exchange the exchange to check for pending find operations
 */
static void
process_find_operations (struct TMH_Exchange *exchange)
{
  struct GNUNET_TIME_Timestamp now;

  now = GNUNET_TIME_timestamp_get ();
  for (struct FeesByWireMethod *fbw = exchange->wire_fees_head;
       NULL != fbw;
       fbw = fbw->next)
  {
    while ( (NULL != fbw->af) &&
            (GNUNET_TIME_timestamp_cmp (fbw->af->end_date,
                                        <,
                                        now)) )
    {
      struct TALER_EXCHANGE_WireAggregateFees *af = fbw->af;

      fbw->af = af->next;
      GNUNET_free (af);
    }
    if (NULL == fbw->af)
    {
      /* Disagreement on the current time */
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  "Exchange has no wire fees configured for `%s' wire method\n",
                  fbw->wire_method);
    }
    else if (GNUNET_TIME_timestamp_cmp (fbw->af->start_date,
                                        >,
                                        now))
    {
      /* Disagreement on the current time */
      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                  "Exchange's earliest fee is %s ahead of our time. Clock skew issue?\n",
                  GNUNET_TIME_relative2s (
                    GNUNET_TIME_absolute_get_remaining (
                      fbw->af->start_date.abs_time),
                    true));
    }
  } /* for all wire methods */

  {
    struct TMH_EXCHANGES_KeysOperation *kon;

    kon = NULL;
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Processing find operations for `%s' (%d)\n",
                exchange->url,
                exchange->state);
    for (struct TMH_EXCHANGES_KeysOperation *ko = exchange->keys_head;
         NULL != ko;
         ko = kon)
    {
      kon = ko->next;
      ko->fc (ko->fc_cls,
              exchange->keys,
              exchange);
      TMH_EXCHANGES_keys4exchange_cancel (ko);
    }
  }
}


/**
 * Function called with information about the wire fees for each wire method.
 * Stores the wire fees within our internal data structures for later use.
 *
 * @param exchange connection to the exchange
 * @param master_pub public key of the exchange
 * @param num_methods number of wire methods supported
 * @param fbm wire fees by method
 * @return #GNUNET_OK on success
 */
static enum GNUNET_GenericReturnValue
process_wire_fees (
  struct TMH_Exchange *exchange,
  const struct TALER_MasterPublicKeyP *master_pub,
  unsigned int num_methods,
  const struct TALER_EXCHANGE_WireFeesByMethod fbm[static num_methods])
{
  for (unsigned int i = 0; i<num_methods; i++)
  {
    const char *wire_method = fbm[i].method;
    const struct TALER_EXCHANGE_WireAggregateFees *fees = fbm[i].fees_head;
    struct FeesByWireMethod *f;
    struct TALER_EXCHANGE_WireAggregateFees *endp;

    for (f = exchange->wire_fees_head; NULL != f; f = f->next)
      if (0 == strcasecmp (wire_method,
                           f->wire_method))
        break;
    if (NULL == f)
    {
      f = GNUNET_new (struct FeesByWireMethod);
      f->wire_method = GNUNET_strdup (wire_method);
      GNUNET_CONTAINER_DLL_insert (exchange->wire_fees_head,
                                   exchange->wire_fees_tail,
                                   f);
    }
    endp = f->af;
    while ( (NULL != endp) &&
            (NULL != endp->next) )
      endp = endp->next;
    while ( (NULL != endp) &&
            (NULL != fees) &&
            (GNUNET_TIME_timestamp_cmp (fees->start_date,
                                        <,
                                        endp->end_date)) )
      fees = fees->next;
    if ( (NULL != endp) &&
         (NULL != fees) &&
         (GNUNET_TIME_timestamp_cmp (fees->start_date,
                                     !=,
                                     endp->end_date)) )
    {
      /* Hole or overlap in the fee structure, not allowed! */
      GNUNET_break_op (0);
      return GNUNET_SYSERR;
    }
    while (NULL != fees)
    {
      struct TALER_EXCHANGE_WireAggregateFees *af;

      af = GNUNET_new (struct TALER_EXCHANGE_WireAggregateFees);
      *af = *fees;
      af->next = NULL;
      if (NULL == endp)
        f->af = af;
      else
        endp->next = af;
      endp = af;
      fees = fees->next;
    } /* all fees for this method */
  } /* for all methods (i) */
  return GNUNET_OK;
}


/**
 * Add account restriction @a a to array of @a restrictions.
 *
 * @param[in,out] restrictions JSON array to build
 * @param r restriction to add to @a restrictions
 * @return #GNUNET_SYSERR if @a r is malformed
 */
static enum GNUNET_GenericReturnValue
add_restriction (json_t *restrictions,
                 const struct TALER_EXCHANGE_AccountRestriction *r)
{
  json_t *jr;

  jr = NULL;
  switch (r->type)
  {
  case TALER_EXCHANGE_AR_INVALID:
    GNUNET_break_op (0);
    return GNUNET_SYSERR;
  case TALER_EXCHANGE_AR_DENY:
    jr = GNUNET_JSON_PACK (
      GNUNET_JSON_pack_string ("type",
                               "deny")
      );
    break;
  case TALER_EXCHANGE_AR_REGEX:
    jr = GNUNET_JSON_PACK (
      GNUNET_JSON_pack_string (
        "type",
        "regex"),
      GNUNET_JSON_pack_string (
        "regex",
        r->details.regex.posix_egrep),
      GNUNET_JSON_pack_string (
        "human_hint",
        r->details.regex.human_hint),
      GNUNET_JSON_pack_object_incref (
        "human_hint_i18n",
        (json_t *) r->details.regex.human_hint_i18n)
      );
    break;
  }
  if (NULL == jr)
  {
    GNUNET_break_op (0);
    return GNUNET_SYSERR;
  }
  GNUNET_assert (0 ==
                 json_array_append_new (restrictions,
                                        jr));
  return GNUNET_OK;

}


/**
 * Retry getting keys from the given exchange in the closure.
 *
 * @param cls the `struct TMH_Exchange *`
 */
static void
retry_exchange (void *cls)
{
  struct TMH_Exchange *exchange = cls;

  exchange->retry_task = NULL;
  GNUNET_assert (NULL == exchange->conn);
  exchange->retry_delay
    = GNUNET_TIME_randomized_backoff (exchange->retry_delay,
                                      RETRY_BACKOFF_THRESHOLD);
  /* Block for the duration of the long-poller */
  exchange->first_retry
    = GNUNET_TIME_relative_to_absolute (LONG_POLL_THRESHOLD);
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Fetching /keys from exchange %s in retry_exchange()\n",
              exchange->url);
  switch (exchange->state)
  {
  case ESTATE_FAILED:
    exchange->state = ESTATE_REDOWNLOADING_FAILURE;
    break;
  case ESTATE_INIT:
    exchange->state = ESTATE_DOWNLOADING_FIRST;
    break;
  case ESTATE_DOWNLOADING_FIRST:
    GNUNET_break (0);
    return;
  case ESTATE_DOWNLOADED:
    exchange->state = ESTATE_REDOWNLOADING_SUCCESS;
    break;
  case ESTATE_REDOWNLOADING_SUCCESS:
    GNUNET_break (0);
    return;
  case ESTATE_REDOWNLOADING_FAILURE:
    GNUNET_break (0);
    return;
  }
  exchange->conn
    = TALER_EXCHANGE_get_keys (
        TMH_curl_ctx,
        exchange->url,
        exchange->keys,
        &keys_mgmt_cb,
        exchange);
  /* Note: while the API spec says 'returns NULL on error', the implementation
     actually never returns NULL. */
  GNUNET_break (NULL != exchange->conn);
}


/**
 * Task to asynchronously return keys operation result to caller.
 *
 * @param cls a `struct TMH_EXCHANGES_KeysOperation`
 */
static void
return_keys (void *cls)
{
  struct TMH_EXCHANGES_KeysOperation *fo = cls;
  struct TMH_Exchange *exchange = fo->my_exchange;

  fo->at = NULL;
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Returning key data for %s instantly\n",
              exchange->url);
  process_find_operations (exchange);
}


struct TMH_EXCHANGES_KeysOperation *
TMH_EXCHANGES_keys4exchange (
  const char *chosen_exchange,
  bool force_download,
  TMH_EXCHANGES_Find2Continuation fc,
  void *fc_cls)
{
  struct TMH_Exchange *exchange;
  struct TMH_EXCHANGES_KeysOperation *fo;

  if (NULL == TMH_curl_ctx)
  {
    GNUNET_break (0);
    return NULL;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Trying to find chosen exchange `%s'\n",
              chosen_exchange);
  exchange = lookup_exchange (chosen_exchange);
  fo = GNUNET_new (struct TMH_EXCHANGES_KeysOperation);
  fo->fc = fc;
  fo->fc_cls = fc_cls;
  fo->my_exchange = exchange;
  GNUNET_CONTAINER_DLL_insert (exchange->keys_head,
                               exchange->keys_tail,
                               fo);
  if ( (NULL != exchange->keys) &&
       (! force_download) &&
       (GNUNET_TIME_absolute_is_future (
          exchange->keys->key_data_expiration.abs_time)) )
  {
    /* We have a valid reply, immediately return result */
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "The exchange `%s' is ready\n",
                exchange->url);
    GNUNET_assert (NULL == fo->at);
    fo->at = GNUNET_SCHEDULER_add_now (&return_keys,
                                       fo);
    return fo;
  }
  if ( (NULL == exchange->conn) &&
       ( (ESTATE_FAILED == exchange->state) ||
         (ESTATE_REDOWNLOADING_FAILURE == exchange->state) ) )
  {
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Already waiting for `%skeys' for a while, failing query instantly\n",
                exchange->url);
    GNUNET_assert (NULL == fo->at);
    fo->at = GNUNET_SCHEDULER_add_now (&return_keys,
                                       fo);
    return fo;
  }
  if ( (force_download) &&
       (GNUNET_TIME_absolute_is_future (exchange->first_retry)) &&
       (ESTATE_DOWNLOADED == exchange->state) )
  {
    /* Return results immediately. */
    fo->at = GNUNET_SCHEDULER_add_now (&return_keys,
                                       fo);
    /* *no* return here, we MAY schedule a 'retry_task' in the
       next block if there isn't one yet */
  }
  if ( (NULL == exchange->retry_task) &&
       (NULL == exchange->conn) )
    exchange->retry_task
      = GNUNET_SCHEDULER_add_at (exchange->first_retry,
                                 &retry_exchange,
                                 exchange);
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Next %skeys (%d) request scheduled for %s\n",
              exchange->url,
              exchange->state,
              GNUNET_TIME_absolute2s (
                exchange->first_retry));
  /* No activity to launch, we are already doing so. */
  return fo;
}


void
TMH_EXCHANGES_keys4exchange_cancel (
  struct TMH_EXCHANGES_KeysOperation *fo)
{
  struct TMH_Exchange *exchange = fo->my_exchange;

  if (NULL != fo->at)
  {
    GNUNET_SCHEDULER_cancel (fo->at);
    fo->at = NULL;
  }
  GNUNET_CONTAINER_DLL_remove (exchange->keys_head,
                               exchange->keys_tail,
                               fo);
  GNUNET_free (fo);
}


/**
 * Obtain applicable fees for @a exchange and @a wire_method.
 *
 * @param exchange the exchange to query
 * @param now current time
 * @param wire_method the wire method we want the fees for
 * @return NULL if we do not have fees for this method yet
 */
static const struct FeesByWireMethod *
get_wire_fees (const struct TMH_Exchange *exchange,
               struct GNUNET_TIME_Timestamp now,
               const char *wire_method)
{
  for (struct FeesByWireMethod *fbw = exchange->wire_fees_head;
       NULL != fbw;
       fbw = fbw->next)
  {
    if (0 == strcasecmp (fbw->wire_method,
                         wire_method) )
    {
      struct TALER_EXCHANGE_WireAggregateFees *af;

      /* Advance through list up to current time */
      while ( (NULL != (af = fbw->af)) &&
              (GNUNET_TIME_timestamp_cmp (now,
                                          >=,
                                          af->end_date)) )
      {
        fbw->af = af->next;
        GNUNET_free (af);
      }
      return fbw;
    }
    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                "Exchange supports `%s' as a wire method (but we do not use that one)\n",
                fbw->wire_method);
  }
  return NULL;
}


/**
 * Free @a exchange.
 *
 * @param[in] exchange entry to free
 */
static void
free_exchange_entry (struct TMH_Exchange *exchange)
{
  struct FeesByWireMethod *f;

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Releasing %s exchange %s (%d)\n",
              exchange->trusted ? "trusted" : "untrusted",
              exchange->url,
              exchange->state);
  GNUNET_CONTAINER_DLL_remove (exchange_head,
                               exchange_tail,
                               exchange);
  purge_exchange_accounts (exchange);
  while (NULL != (f = exchange->wire_fees_head))
  {
    struct TALER_EXCHANGE_WireAggregateFees *af;

    GNUNET_CONTAINER_DLL_remove (exchange->wire_fees_head,
                                 exchange->wire_fees_tail,
                                 f);
    while (NULL != (af = f->af))
    {
      f->af = af->next;
      GNUNET_free (af);
    }
    GNUNET_free (f->wire_method);
    GNUNET_free (f);
  }
  if (NULL != exchange->conn)
  {
    TALER_EXCHANGE_get_keys_cancel (exchange->conn);
    exchange->conn = NULL;
  }
  TALER_EXCHANGE_keys_decref (exchange->keys);
  exchange->keys = NULL;
  if (NULL != exchange->retry_task)
  {
    GNUNET_SCHEDULER_cancel (exchange->retry_task);
    exchange->retry_task = NULL;
  }
  GNUNET_assert (NULL == exchange->keys_head);
  GNUNET_assert (NULL == exchange->keys_tail);
  GNUNET_free (exchange->currency);
  GNUNET_free (exchange->url);
  GNUNET_free (exchange);
}


/**
 * We failed downloading /keys from @a exchange. Tell clients
 * about our failure, abort pending operations and retry later.
 *
 * @param exchange exchange that failed
 */
static void
fail_and_retry (struct TMH_Exchange *exchange)
{
  struct TMH_EXCHANGES_KeysOperation *keys;

  exchange->state = ESTATE_FAILED;
  while (NULL != (keys = exchange->keys_head))
  {
    keys->fc (keys->fc_cls,
              NULL,
              exchange);
    TMH_EXCHANGES_keys4exchange_cancel (keys);
  }
  exchange->retry_delay
    = GNUNET_TIME_randomized_backoff (exchange->retry_delay,
                                      RETRY_BACKOFF_THRESHOLD);
  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
              "Failed to fetch /keys from `%s'; retrying in %s\n",
              exchange->url,
              GNUNET_STRINGS_relative_time_to_string (exchange->retry_delay,
                                                      true));
  if (NULL != exchange->retry_task)
    GNUNET_SCHEDULER_cancel (exchange->retry_task);
  exchange->first_retry
    = GNUNET_TIME_relative_to_absolute (
        exchange->retry_delay);
  exchange->retry_task
    = GNUNET_SCHEDULER_add_delayed (exchange->retry_delay,
                                    &retry_exchange,
                                    exchange);
}


/**
 * Update our information in the database about the
 * /keys of an exchange. Run inside of a database
 * transaction scope that will re-try and/or commit
 * depending on the return value.
 *
 * @param keys information to persist
 * @return transaction status
 */
static enum GNUNET_DB_QueryStatus
insert_keys_data (const struct TALER_EXCHANGE_Keys *keys)
{
  enum GNUNET_DB_QueryStatus qs;

  /* store exchange online signing keys in our DB */
  for (unsigned int i = 0; i<keys->num_sign_keys; i++)
  {
    struct TALER_EXCHANGE_SigningPublicKey *sign_key = &keys->sign_keys[i];
    enum GNUNET_DB_QueryStatus qs;

    qs = TMH_db->insert_exchange_signkey (
      TMH_db->cls,
      &keys->master_pub,
      &sign_key->key,
      sign_key->valid_from,
      sign_key->valid_until,
      sign_key->valid_legal,
      &sign_key->master_sig);
    /* 0 is OK, we may already have the key in the DB! */
    if (0 > qs)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      return qs;
    }
  }

  qs = TMH_db->insert_exchange_keys (TMH_db->cls,
                                     keys);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    return qs;
  }

  qs = TMH_db->delete_exchange_accounts (TMH_db->cls,
                                         &keys->master_pub);
  if (0 > qs)
  {
    GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
    return qs;
  }

  for (unsigned int i = 0; i<keys->accounts_len; i++)
  {
    const struct TALER_EXCHANGE_WireAccount *account = &keys->accounts[i];
    json_t *debit_restrictions;
    json_t *credit_restrictions;

    debit_restrictions = json_array ();
    GNUNET_assert (NULL != debit_restrictions);
    credit_restrictions = json_array ();
    GNUNET_assert (NULL != credit_restrictions);
    for (unsigned int j = 0; j<account->debit_restrictions_length; j++)
    {
      if (GNUNET_OK !=
          add_restriction (debit_restrictions,
                           &account->debit_restrictions[j]))
      {
        TMH_db->rollback (TMH_db->cls);
        GNUNET_break (0);
        json_decref (debit_restrictions);
        json_decref (credit_restrictions);
        return GNUNET_DB_STATUS_HARD_ERROR;
      }
    }
    for (unsigned int j = 0; j<account->credit_restrictions_length; j++)
    {
      if (GNUNET_OK !=
          add_restriction (credit_restrictions,
                           &account->credit_restrictions[j]))
      {
        TMH_db->rollback (TMH_db->cls);
        GNUNET_break (0);
        json_decref (debit_restrictions);
        json_decref (credit_restrictions);
        return GNUNET_DB_STATUS_HARD_ERROR;
      }
    }
    qs = TMH_db->insert_exchange_account (
      TMH_db->cls,
      &keys->master_pub,
      account->payto_uri,
      account->conversion_url,
      debit_restrictions,
      credit_restrictions,
      &account->master_sig);
    json_decref (debit_restrictions);
    json_decref (credit_restrictions);
    if (qs < 0)
    {
      GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
      return qs;
    }
  } /* end 'for all accounts' */

  for (unsigned int i = 0; i<keys->fees_len; i++)
  {
    const struct TALER_EXCHANGE_WireFeesByMethod *fbm = &keys->fees[i];
    const char *wire_method = fbm->method;
    const struct TALER_EXCHANGE_WireAggregateFees *fees = fbm->fees_head;

    while (NULL != fees)
    {
      struct GNUNET_HashCode h_wire_method;

      GNUNET_CRYPTO_hash (wire_method,
                          strlen (wire_method) + 1,
                          &h_wire_method);
      qs = TMH_db->store_wire_fee_by_exchange (TMH_db->cls,
                                               &keys->master_pub,
                                               &h_wire_method,
                                               &fees->fees,
                                               fees->start_date,
                                               fees->end_date,
                                               &fees->master_sig);
      if (0 > qs)
      {
        GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
        return qs;
      }
      fees = fees->next;
    } /* all fees for this method */
  } /* for all methods (i) */

  {
    struct GNUNET_DB_EventHeaderP es = {
      .size = ntohs (sizeof (es)),
      .type = ntohs (TALER_DBEVENT_MERCHANT_EXCHANGE_KEYS)
    };

    TMH_db->event_notify (TMH_db->cls,
                          &es,
                          keys->exchange_url,
                          strlen (keys->exchange_url) + 1);
  }
  return qs;
}


static void
keys_mgmt_cb (void *cls,
              const struct TALER_EXCHANGE_KeysResponse *kr,
              struct TALER_EXCHANGE_Keys *keys)
{
  struct TMH_Exchange *exchange = cls;
  enum GNUNET_DB_QueryStatus qs;

  exchange->conn = NULL;
  if (MHD_HTTP_OK != kr->hr.http_status)
  {
    if (GNUNET_TIME_absolute_is_future (exchange->first_retry))
    {
      /* /keys failed *before* the long polling threshold.
         We apply the exponential back-off from now. */
      exchange->first_retry
        = GNUNET_TIME_relative_to_absolute (
            exchange->retry_delay);
    }
    fail_and_retry (exchange);
    TALER_EXCHANGE_keys_decref (keys);
    return;
  }
  if (NULL == exchange->currency)
    exchange->currency = GNUNET_strdup (keys->currency);
  if (0 != strcmp (exchange->currency,
                   keys->currency))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "/keys response from `%s' is for currency `%s', but we expected `%s'\n",
                exchange->url,
                keys->currency,
                exchange->currency);
    fail_and_retry (exchange);
    TALER_EXCHANGE_keys_decref (keys);
    return;
  }
  exchange->state = ESTATE_DOWNLOADED;
  TMH_db->preflight (TMH_db->cls);
  for (unsigned int r = 0; r<MAX_RETRIES; r++)
  {
    if (GNUNET_OK !=
        TMH_db->start (TMH_db->cls,
                       "update exchange key data"))
    {
      TMH_db->rollback (TMH_db->cls);
      GNUNET_break (0);
      fail_and_retry (exchange);
      TALER_EXCHANGE_keys_decref (keys);
      return;
    }

    qs = insert_keys_data (keys);
    if (qs < 0)
    {
      TMH_db->rollback (TMH_db->cls);
      if (GNUNET_DB_STATUS_SOFT_ERROR == qs)
        continue;
      GNUNET_break (0);
      fail_and_retry (exchange);
      TALER_EXCHANGE_keys_decref (keys);
      return;
    }

    qs = TMH_db->commit (TMH_db->cls);
    if (qs < 0)
    {
      TMH_db->rollback (TMH_db->cls);
      if (GNUNET_DB_STATUS_SOFT_ERROR == qs)
        continue;
      GNUNET_break (0);
      fail_and_retry (exchange);
      TALER_EXCHANGE_keys_decref (keys);
      return;
    }
  } /* end of retry loop */
  if (qs < 0)
  {
    GNUNET_break (0);
    fail_and_retry (exchange);
    TALER_EXCHANGE_keys_decref (keys);
    return;
  }
  TALER_EXCHANGE_keys_decref (keys);
  exchange->retry_delay = GNUNET_TIME_UNIT_ZERO;
}


enum GNUNET_GenericReturnValue
TMH_EXCHANGES_lookup_wire_fee (
  const struct TMH_Exchange *exchange,
  const char *wire_method,
  struct TALER_Amount *wire_fee)
{
  const struct FeesByWireMethod *fbm;
  const struct TALER_EXCHANGE_WireAggregateFees *af;

  fbm = get_wire_fees (exchange,
                       GNUNET_TIME_timestamp_get (),
                       wire_method);
  if (NULL == fbm)
    return GNUNET_NO;
  af = fbm->af;
  *wire_fee = af->fees.wire;
  return GNUNET_OK;
}


enum GNUNET_GenericReturnValue
TMH_exchange_check_debit (
  const struct TMH_Exchange *exchange,
  const struct TMH_WireMethod *wm)
{
  if (NULL == exchange->acc_head)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "No accounts available for %s\n",
                exchange->url);
    return GNUNET_SYSERR;
  }
  for (struct ExchangeAccount *acc = exchange->acc_head;
       NULL != acc;
       acc = acc->next)
  {
    bool ok = true;

    if (0 != strcmp (acc->wire_method,
                     wm->wire_method))
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  "Exchange %s wire method %s != %s\n",
                  exchange->url,
                  acc->wire_method,
                  wm->wire_method);
      continue;
    }
    if (NULL != acc->conversion_url)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  "Exchange %s account requires currency conversion (not supported)\n",
                  exchange->url);
      continue; /* never use accounts with conversion */
    }
    for (struct Restriction *r = acc->d_head;
         NULL != r;
         r = r->next)
    {
      switch (r->type)
      {
      case TALER_EXCHANGE_AR_INVALID:
        GNUNET_break (0);
        ok = false;
        break;
      case TALER_EXCHANGE_AR_DENY:
        ok = false;
        GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                    "Exchange %s account is disabled\n",
                    exchange->url);
        break;
      case TALER_EXCHANGE_AR_REGEX:
        if (0 != regexec (&r->details.regex.ex,
                          wm->payto_uri,
                          0, NULL, 0))
        {
          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                      "Exchange %s account regex does not match %s\n",
                      exchange->url,
                      wm->payto_uri);
          ok = false;
        }
        break;
      }
      if (! ok)
        break;
    }

    if (ok)
      return GNUNET_OK;
  }
  return GNUNET_NO;
}


void
TMH_exchange_get_trusted (TMH_ExchangeCallback cb,
                          void *cb_cls)
{
  for (const struct TMH_Exchange *exchange = exchange_head;
       NULL != exchange;
       exchange = exchange->next)
  {
    if (! exchange->trusted)
    {
      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                  "Exchange %s not trusted, skipping!\n",
                  exchange->url);
      continue;
    }
    cb (cb_cls,
        exchange->url,
        exchange);
  }
}


bool
TMH_test_exchange_configured_for_currency (
  const char *currency)
{
  for (const struct TMH_Exchange *exchange = exchange_head;
       NULL != exchange;
       exchange = exchange->next)
  {
    if (! exchange->trusted)
      continue;
    if (NULL == exchange->currency)
      continue;
    if (0 == strcmp (currency,
                     exchange->currency))
      return true;
  }
  return false;
}


/**
 * Function called on each configuration section. Finds sections
 * about exchanges, parses the entries.
 *
 * @param cls closure, with a `const struct GNUNET_CONFIGURATION_Handle *`
 * @param section name of the section
 */
static void
accept_exchanges (void *cls,
                  const char *section)
{
  const struct GNUNET_CONFIGURATION_Handle *cfg = cls;
  char *url;
  char *mks;
  struct TMH_Exchange *exchange;
  char *currency;

  if (GNUNET_SYSERR == trusted_exchange_count)
    return;
  if (0 != strncasecmp (section,
                        "merchant-exchange-",
                        strlen ("merchant-exchange-")))
    return;
  if (GNUNET_YES ==
      GNUNET_CONFIGURATION_get_value_yesno (cfg,
                                            section,
                                            "DISABLED"))
    return;
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_string (cfg,
                                             section,
                                             "EXCHANGE_BASE_URL",
                                             &url))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                               section,
                               "EXCHANGE_BASE_URL");
    return;
  }
  if (GNUNET_OK !=
      GNUNET_CONFIGURATION_get_value_string (cfg,
                                             section,
                                             "CURRENCY",
                                             &currency))
  {
    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                               section,
                               "CURRENCY");
    GNUNET_free (url);
    return;
  }
  exchange = lookup_exchange (url);
  GNUNET_free (url);
  if (NULL == exchange->currency)
    exchange->currency = currency;
  else
    GNUNET_free (currency);
  if (GNUNET_OK ==
      GNUNET_CONFIGURATION_get_value_string (cfg,
                                             section,
                                             "MASTER_KEY",
                                             &mks))
  {
    if (GNUNET_OK ==
        GNUNET_CRYPTO_eddsa_public_key_from_string (
          mks,
          strlen (mks),
          &exchange->master_pub.eddsa_pub))
    {
      exchange->trusted = true;
      trusted_exchange_count++;
    }
    else
    {
      GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
                                 section,
                                 "MASTER_KEY",
                                 "malformed EdDSA key");
    }
    GNUNET_free (mks);
  }
  else
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "MASTER_KEY missing in section '%s', not trusting exchange\n",
                section);
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Setup exchange %s as %s\n",
              exchange->url,
              exchange->trusted ? "trusted" : "untrusted");
  if (NULL != exchange->retry_task)
  {
    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                "Exchange at `%s' configured in multiple configuration sections (see `%s')!\n",
                exchange->url,
                section);
    trusted_exchange_count = GNUNET_SYSERR;
    return;
  }
  exchange->retry_task
    = GNUNET_SCHEDULER_add_now (&retry_exchange,
                                exchange);
}


/**
 * Trigger (re)loading of keys from DB.
 *
 * @param cls NULL
 * @param extra base URL of the exchange that changed
 * @param extra_len number of bytes in @a extra
 */
static void
update_exchange_keys (void *cls,
                      const void *extra,
                      size_t extra_len)
{
  enum GNUNET_DB_QueryStatus qs;
  const char *url = extra;
  struct TMH_Exchange *exchange;
  struct TALER_EXCHANGE_Keys *keys;

  if ( (NULL == extra) ||
       (0 == extra_len) )
  {
    GNUNET_break (0);
    return;
  }
  if ('\0' != url[extra_len - 1])
  {
    GNUNET_break (0);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Received keys change notification: reload `%s'\n",
              url);
  exchange = lookup_exchange (url);
  qs = TMH_db->select_exchange_keys (TMH_db->cls,
                                     url,
                                     &keys);
  if (qs <= 0)
  {
    GNUNET_break (0);
    return;
  }
  if (NULL == exchange->currency)
    exchange->currency = GNUNET_strdup (keys->currency);
  if (0 != strcmp (keys->currency,
                   exchange->currency))
  {
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "/keys cached in our database are for currency `%s', but we expected `%s'\n",
                keys->currency,
                exchange->currency);
    return;
  }
  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Loaded /keys from database with %u accounts\n",
              keys->accounts_len);
  if (GNUNET_OK !=
      set_exchange_accounts (exchange,
                             keys->accounts_len,
                             keys->accounts))
  {
    /* invalid account specification given */
    GNUNET_break_op (0);
    /* but: we can continue anyway, things may just not
       work, but probably better than to not keep going. */
  }
  if (GNUNET_OK !=
      process_wire_fees (exchange,
                         &keys->master_pub,
                         keys->fees_len,
                         keys->fees))
  {
    /* invalid wire fee specification given */
    GNUNET_break_op (0);
    /* but: we can continue anyway, things may just not
       work, but probably better than to not keep going. */
    return;
  }

  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
              "Reloaded /keys of %s from database\n",
              url);
  TALER_EXCHANGE_keys_decref (exchange->keys);
  exchange->keys = keys;
  if ( (exchange->trusted) &&
       (0 != GNUNET_memcmp (&exchange->master_pub,
                            &keys->master_pub)) )
  {
    /* master pub differs => do not trust the exchange (without auditor) */
    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                "Master public key of exchange `%s' differs from our configuration. Not trusting exchange.\n",
                exchange->url);
    exchange->trusted = false;
  }
  if (! exchange->trusted)
  {
    exchange->master_pub = keys->master_pub;
    for (struct TMH_Exchange *e = exchange_head;
         NULL != e;
         e = e->next)
    {
      if (e == exchange)
        continue;
      if (! e->trusted)
        continue;
      if (0 ==
          GNUNET_memcmp (&e->master_pub,
                         &exchange->master_pub))
        exchange->trusted = true; /* same exchange, different URL => trust applies */
    }
  }

  process_find_operations (exchange);
}


enum GNUNET_GenericReturnValue
TMH_EXCHANGES_init (const struct GNUNET_CONFIGURATION_Handle *cfg)
{
  /* get exchanges from the merchant configuration and try to connect to them */
  {
    struct GNUNET_DB_EventHeaderP es = {
      .size = ntohs (sizeof (es)),
      .type = ntohs (TALER_DBEVENT_MERCHANT_EXCHANGE_KEYS)
    };

    keys_eh = TMH_db->event_listen (TMH_db->cls,
                                    &es,
                                    GNUNET_TIME_UNIT_FOREVER_REL,
                                    &update_exchange_keys,
                                    NULL);
  }
  GNUNET_CONFIGURATION_iterate_sections (cfg,
                                         &accept_exchanges,
                                         (void *) cfg);
  /* build JSON with list of trusted exchanges (will be included in contracts) */
  return trusted_exchange_count;
}


void
TMH_EXCHANGES_done ()
{
  if (NULL != keys_eh)
  {
    TMH_db->event_listen_cancel (keys_eh);
    keys_eh = NULL;
  }
  while (NULL != exchange_head)
    free_exchange_entry (exchange_head);
}


/* end of taler-merchant-httpd_exchanges.c */