exchange

secmod_cs.c (66649B)

---

 /*
   This file is part of TALER
   Copyright (C) 2014-2026 Taler Systems SA
 
   TALER is free software; you can redistribute it and/or modify it under the
   terms of the GNU General Public License as published by the Free Software
   Foundation; either version 3, or (at your option) any later version.
 
   TALER is distributed in the hope that it will be useful, but WITHOUT ANY
   WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
   A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
 
   You should have received a copy of the GNU General Public License along with
   TALER; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
 */
 /**
  * @file util/secmod_cs.c
  * @brief Standalone process to perform private key CS operations
  * @author Christian Grothoff
  *
  * Key design points:
  * - EVERY thread of the exchange will have its own pair of connections to the
  *   crypto helpers.  This way, every thread will also have its own /keys state
  *   and avoid the need to synchronize on those.
  * - auditor signatures and master signatures are to be kept in the exchange DB,
  *   and merged with the public keys of the helper by the exchange HTTPD!
  * - the main loop of the helper is SINGLE-THREADED, but there are
  *   threads for crypto-workers which do the signing in parallel, one per client.
  * - thread-safety: signing happens in parallel, thus when REMOVING private keys,
  *   we must ensure that all signers are done before we fully free() the
  *   private key. This is done by reference counting (as work is always
  *   assigned and collected by the main thread).
  */
 #include "platform.h"
 #include "taler/taler_util.h"
 #include "secmod_cs.h"
 #include <gcrypt.h>
 #include <pthread.h>
 #include <sys/eventfd.h>
 #include "taler/taler_error_codes.h"
 #include "taler/taler_signatures.h"
 #include "secmod_common.h"
 #include <poll.h>
 
 
 /**
  * Information we keep per denomination.
  */
 struct Denomination;
 
 
 /**
  * One particular denomination key.
  */
 struct DenominationKey
 {
 
   /**
    * Kept in a DLL of the respective denomination. Sorted by anchor time.
    */
   struct DenominationKey *next;
 
   /**
    * Kept in a DLL of the respective denomination. Sorted by anchor time.
    */
   struct DenominationKey *prev;
 
   /**
    * Denomination this key belongs to.
    */
   struct Denomination *denom;
 
   /**
    * Name of the file this key is stored under.
    */
   char *filename;
 
   /**
    * The private key of the denomination.
    */
   struct GNUNET_CRYPTO_CsPrivateKey denom_priv;
 
   /**
    * The public key of the denomination.
    */
   struct GNUNET_CRYPTO_CsPublicKey denom_pub;
 
   /**
    * Message to transmit to clients to introduce this public key.
    */
   struct TALER_CRYPTO_CsKeyAvailableNotification *an;
 
   /**
    * Hash of this denomination's public key.
    */
   struct TALER_CsPubHashP h_cs;
 
   /**
    * Time at which this key is supposed to become valid.
    */
   struct GNUNET_TIME_Timestamp anchor_start;
 
   /**
    * Time at which this key is supposed to expire (exclusive).
    */
   struct GNUNET_TIME_Timestamp anchor_end;
 
   /**
    * Generation when this key was created or revoked.
    */
   uint64_t key_gen;
 
   /**
    * Reference counter. Counts the number of threads that are
    * using this key at this time.
    */
   unsigned int rc;
 
   /**
    * Flag set to true if this key has been purged and the memory
    * must be freed as soon as @e rc hits zero.
    */
   bool purge;
 
 };
 
 
 struct Denomination
 {
 
   /**
    * Kept in a DLL. Sorted by #denomination_action_time().
    */
   struct Denomination *next;
 
   /**
    * Kept in a DLL. Sorted by #denomination_action_time().
    */
   struct Denomination *prev;
 
   /**
    * Head of DLL of actual keys of this denomination.
    */
   struct DenominationKey *keys_head;
 
   /**
    * Tail of DLL of actual keys of this denomination.
    */
   struct DenominationKey *keys_tail;
 
   /**
    * How long can coins be withdrawn (generated)?  Should be small
    * enough to limit how many coins will be signed into existence with
    * the same key, but large enough to still provide a reasonable
    * anonymity set.
    */
   struct GNUNET_TIME_Relative duration_withdraw;
 
   /**
    * What is the configuration section of this denomination type?  Also used
    * for the directory name where the denomination keys are stored.
    */
   char *section;
 
 };
 
 
 /**
  * A semaphore.
  */
 struct Semaphore
 {
   /**
    * Mutex for the semaphore.
    */
   pthread_mutex_t mutex;
 
   /**
    * Condition variable for the semaphore.
    */
   pthread_cond_t cv;
 
   /**
    * Counter of the semaphore.
    */
   unsigned int ctr;
 };
 
 
 /**
  * Job in a batch sign request.
  */
 struct BatchJob;
 
 /**
  * Handle for a thread that does work in batch signing.
  */
 struct Worker
 {
   /**
    * Kept in a DLL.
    */
   struct Worker *prev;
 
   /**
    * Kept in a DLL.
    */
   struct Worker *next;
 
   /**
    * Job this worker should do next.
    */
   struct BatchJob *job;
 
   /**
    * Semaphore to signal the worker that a job is available.
    */
   struct Semaphore sem;
 
   /**
    * Handle for this thread.
    */
   pthread_t pt;
 
   /**
    * Set to true if the worker should terminate.
    */
   bool do_shutdown;
 };
 
 
 /**
  * Job in a batch sign request.
  */
 struct BatchJob
 {
 
   /**
    * Thread doing the work.
    */
   struct Worker *worker;
 
   /**
    * Semaphore to signal that the job is finished.
    */
   struct Semaphore sem;
 
   /**
    * Computation status.
    */
   enum TALER_ErrorCode ec;
 
   /**
    * Which type of request is this?
    */
   enum { TYPE_SIGN, TYPE_RDERIVE } type;
 
   /**
    * Details depending on @e type.
    */
   union
   {
 
     /**
      * Details if @e type is TYPE_SIGN.
      */
     struct
     {
       /**
        * Request we are working on.
        */
       const struct TALER_CRYPTO_CsSignRequestMessage *sr;
 
       /**
        * Result with the signature.
        */
       struct GNUNET_CRYPTO_CsBlindSignature cs_answer;
     } sign;
 
     /**
      * Details if type is TYPE_RDERIVE.
      */
     struct
     {
       /**
        * Request we are answering.
        */
       const struct TALER_CRYPTO_CsRDeriveRequest *rdr;
 
       /**
        * Pair of points to return.
        */
       struct GNUNET_CRYPTO_CSPublicRPairP rpairp;
 
     } rderive;
 
   } details;
 
 };
 
 /**
  * Head of DLL of workers ready for more work.
  */
 static struct Worker *worker_head;
 
 /**
  * Tail of DLL of workers ready for more work.
  */
 static struct Worker *worker_tail;
 
 /**
  * Lock for manipulating the worker DLL.
  */
 static pthread_mutex_t worker_lock;
 
 /**
  * Total number of workers that were started.
  */
 static unsigned int workers;
 
 /**
  * Semaphore used to grab a worker.
  */
 static struct Semaphore worker_sem;
 
 /**
  * Command-line options for various TALER_SECMOD_XXX_run() functions.
  */
 static struct TALER_SECMOD_Options *globals;
 
 /**
  * Where do we store the keys?
  */
 static char *keydir;
 
 /**
  * How much should coin creation (@e duration_withdraw) duration overlap
  * with the next denomination?  Basically, the starting time of two
  * denominations is always @e duration_withdraw - #overlap_duration apart.
  */
 static struct GNUNET_TIME_Relative overlap_duration;
 
 /**
  * How long into the future do we pre-generate keys?
  */
 static struct GNUNET_TIME_Relative lookahead_sign;
 
 /**
  * All of our denominations, in a DLL. Sorted?
  */
 static struct Denomination *denom_head;
 
 /**
  * All of our denominations, in a DLL. Sorted?
  */
 static struct Denomination *denom_tail;
 
 /**
  * Map of hashes of public (CS) keys to `struct DenominationKey *`
  * with the respective private keys.
  */
 static struct GNUNET_CONTAINER_MultiHashMap *keys;
 
 /**
  * Task run to generate new keys.
  */
 static struct GNUNET_SCHEDULER_Task *keygen_task;
 
 /**
  * Lock for the keys queue.
  */
 static pthread_mutex_t keys_lock;
 
 /**
  * Current key generation.
  */
 static uint64_t key_gen;
 
 /**
  * Generate the announcement message for @a dk.
  *
  * @param[in,out] dk denomination key to generate the announcement for
  */
 static void
 generate_response (struct DenominationKey *dk)
 {
   struct Denomination *denom = dk->denom;
   size_t nlen = strlen (denom->section) + 1;
   struct TALER_CRYPTO_CsKeyAvailableNotification *an;
   void *p;
   size_t tlen;
   struct GNUNET_TIME_Relative effective_duration;
 
   GNUNET_assert (sizeof(dk->denom_pub) < UINT16_MAX);
   GNUNET_assert (nlen < UINT16_MAX);
   tlen = nlen + sizeof (*an);
   GNUNET_assert (tlen < UINT16_MAX);
   an = GNUNET_malloc (tlen);
   an->header.size = htons ((uint16_t) tlen);
   an->header.type = htons (TALER_HELPER_CS_MT_AVAIL);
   an->section_name_len = htons ((uint16_t) nlen);
   an->anchor_time = GNUNET_TIME_timestamp_hton (dk->anchor_start);
   effective_duration = GNUNET_TIME_absolute_get_difference (
     dk->anchor_start.abs_time,
     dk->anchor_end.abs_time);
   an->duration_withdraw = GNUNET_TIME_relative_hton (effective_duration);
   an->denom_pub = dk->denom_pub;
   TALER_exchange_secmod_cs_sign (&dk->h_cs,
                                  denom->section,
                                  dk->anchor_start,
                                  effective_duration,
                                  &TES_smpriv,
                                  &an->secm_sig);
   an->secm_pub = TES_smpub;
   p = (void *) &an[1];
   GNUNET_memcpy (p,
                  denom->section,
                  nlen);
   dk->an = an;
 }
 
 
 /**
  * Do the actual signing work.
  *
  * @param h_cs hash of key to sign with
  * @param planchet message to sign
  * @param for_melt true if for melting
  * @param[out] cs_sigp set to the CS signature
  * @return #TALER_EC_NONE on success
  */
 static enum TALER_ErrorCode
 do_sign (const struct TALER_CsPubHashP *h_cs,
          const struct GNUNET_CRYPTO_CsBlindedMessage *planchet,
          bool for_melt,
          struct GNUNET_CRYPTO_CsBlindSignature *cs_sigp)
 {
   struct GNUNET_CRYPTO_CsRSecret r[2];
   struct DenominationKey *dk;
 
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   dk = GNUNET_CONTAINER_multihashmap_get (keys,
                                           &h_cs->hash);
   if (NULL == dk)
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Signing request failed, denomination key %s unknown\n",
                 GNUNET_h2s (&h_cs->hash));
     return TALER_EC_EXCHANGE_GENERIC_DENOMINATION_KEY_UNKNOWN;
   }
   if (GNUNET_TIME_absolute_is_future (dk->anchor_start.abs_time))
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Signing request failed, denomination key %s is not yet valid\n",
                 GNUNET_h2s (&h_cs->hash));
     return TALER_EC_EXCHANGE_DENOMINATION_HELPER_TOO_EARLY;
   }
   if (GNUNET_TIME_absolute_is_past (dk->anchor_end.abs_time))
   {
     /* it is too late; now, usually we should never get here
        as we delete upon expiration, so this is just conservative */
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Signing request failed, denomination key %s is expired (%llu)\n",
                 GNUNET_h2s (&h_cs->hash),
                 (unsigned long long) dk->anchor_end.abs_time.abs_value_us);
     /* usually we delete upon expiratoin, hence same EC */
     return TALER_EC_EXCHANGE_GENERIC_DENOMINATION_KEY_UNKNOWN;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Received request to sign over bytes with key %s\n",
               GNUNET_h2s (&h_cs->hash));
   GNUNET_assert (dk->rc < UINT_MAX);
   dk->rc++;
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   GNUNET_CRYPTO_cs_r_derive (&planchet->nonce,
                              for_melt ? "rm" : "rw",
                              &dk->denom_priv,
                              r);
   GNUNET_CRYPTO_cs_sign_derive (&dk->denom_priv,
                                 r,
                                 planchet,
                                 cs_sigp);
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   GNUNET_assert (dk->rc > 0);
   dk->rc--;
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   return TALER_EC_NONE;
 }
 
 
 /**
  * Generate error response that signing failed.
  *
  * @param client client to send response to
  * @param ec error code to include
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 fail_sign (struct TES_Client *client,
            enum TALER_ErrorCode ec)
 {
   struct TALER_CRYPTO_SignFailure sf = {
     .header.size = htons (sizeof (sf)),
     .header.type = htons (TALER_HELPER_CS_MT_RES_SIGN_FAILURE),
     .ec = htonl (ec)
   };
 
   return TES_transmit (client->csock,
                        &sf.header);
 }
 
 
 /**
  * Generate error response that deriving failed.
  *
  * @param client client to send response to
  * @param ec error code to include
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 fail_derive (struct TES_Client *client,
              enum TALER_ErrorCode ec)
 {
   struct TALER_CRYPTO_RDeriveFailure sf = {
     .header.size = htons (sizeof (sf)),
     .header.type = htons (TALER_HELPER_CS_MT_RES_RDERIVE_FAILURE),
     .ec = htonl (ec)
   };
 
   return TES_transmit (client->csock,
                        &sf.header);
 }
 
 
 /**
  * Generate signature response.
  *
  * @param client client to send response to
  * @param cs_answer signature to send
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 send_signature (struct TES_Client *client,
                 const struct GNUNET_CRYPTO_CsBlindSignature *cs_answer)
 {
   struct TALER_CRYPTO_SignResponse sres;
 
   sres.header.size = htons (sizeof (sres));
   sres.header.type = htons (TALER_HELPER_CS_MT_RES_SIGNATURE);
   sres.b = htonl (cs_answer->b);
   sres.cs_answer = cs_answer->s_scalar;
   return TES_transmit (client->csock,
                        &sres.header);
 }
 
 
 /**
  * Handle @a client request @a sr to create signature. Create the
  * signature using the respective key and return the result to
  * the client.
  *
  * @param client the client making the request
  * @param sr the request details
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 handle_sign_request (struct TES_Client *client,
                      const struct TALER_CRYPTO_CsSignRequestMessage *sr)
 {
   struct GNUNET_CRYPTO_CsBlindSignature cs_answer;
   struct GNUNET_TIME_Absolute now = GNUNET_TIME_absolute_get ();
   enum TALER_ErrorCode ec;
   enum GNUNET_GenericReturnValue ret;
 
   ec = do_sign (&sr->h_cs,
                 &sr->message,
                 (0 != ntohl (sr->for_melt)),
                 &cs_answer);
   if (TALER_EC_NONE != ec)
   {
     return fail_sign (client,
                       ec);
   }
   ret = send_signature (client,
                         &cs_answer);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Sent CS signature after %s\n",
               GNUNET_TIME_relative2s (
                 GNUNET_TIME_absolute_get_duration (now),
                 GNUNET_YES));
   return ret;
 }
 
 
 /**
  * Do the actual deriving work.
  *
  * @param h_cs key to sign with
  * @param nonce nonce to derive from
  * @param for_melt true if for melting
  * @param[out] rpairp set to the derived values
  * @return #TALER_EC_NONE on success
  */
 static enum TALER_ErrorCode
 do_derive (const struct TALER_CsPubHashP *h_cs,
            const struct GNUNET_CRYPTO_CsSessionNonce *nonce,
            bool for_melt,
            struct GNUNET_CRYPTO_CSPublicRPairP *rpairp)
 {
   struct DenominationKey *dk;
   struct GNUNET_CRYPTO_CSPrivateRPairP r_priv;
 
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   dk = GNUNET_CONTAINER_multihashmap_get (keys,
                                           &h_cs->hash);
   if (NULL == dk)
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "R Derive request failed, denomination key %s unknown\n",
                 GNUNET_h2s (&h_cs->hash));
     return TALER_EC_EXCHANGE_GENERIC_DENOMINATION_KEY_UNKNOWN;
   }
   if (GNUNET_TIME_absolute_is_future (dk->anchor_start.abs_time))
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "R Derive request failed, denomination key %s is not yet valid\n",
                 GNUNET_h2s (&h_cs->hash));
     return TALER_EC_EXCHANGE_DENOMINATION_HELPER_TOO_EARLY;
   }
   if (GNUNET_TIME_absolute_is_past (dk->anchor_end.abs_time))
   {
     /* it is too late; now, usually we should never get here
        as we delete upon expiration, so this is just conservative */
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Signing request failed, denomination key %s is expired (%llu)\n",
                 GNUNET_h2s (&h_cs->hash),
                 (unsigned long long) dk->anchor_end.abs_time.abs_value_us);
     /* usually we delete upon expiratoin, hence same EC */
     return TALER_EC_EXCHANGE_GENERIC_DENOMINATION_KEY_UNKNOWN;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Received request to derive R with key %s\n",
               GNUNET_h2s (&h_cs->hash));
   GNUNET_assert (dk->rc < UINT_MAX);
   dk->rc++;
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   GNUNET_CRYPTO_cs_r_derive (nonce,
                              for_melt ? "rm" : "rw",
                              &dk->denom_priv,
                              r_priv.r);
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   GNUNET_assert (dk->rc > 0);
   dk->rc--;
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   GNUNET_CRYPTO_cs_r_get_public (&r_priv.r[0],
                                  &rpairp->r_pub[0]);
   GNUNET_CRYPTO_cs_r_get_public (&r_priv.r[1],
                                  &rpairp->r_pub[1]);
   return TALER_EC_NONE;
 }
 
 
 /**
  * Generate derivation response.
  *
  * @param client client to send response to
  * @param r_pub public point value pair to send
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 send_derivation (struct TES_Client *client,
                  const struct GNUNET_CRYPTO_CSPublicRPairP *r_pub)
 {
   struct TALER_CRYPTO_RDeriveResponse rdr = {
     .header.size = htons (sizeof (rdr)),
     .header.type = htons (TALER_HELPER_CS_MT_RES_RDERIVE),
     .r_pub = *r_pub
   };
 
   return TES_transmit (client->csock,
                        &rdr.header);
 }
 
 
 /**
  * Initialize a semaphore @a sem with a value of @a val.
  *
  * @param[out] sem semaphore to initialize
  * @param val initial value of the semaphore
  */
 static void
 sem_init (struct Semaphore *sem,
           unsigned int val)
 {
   GNUNET_assert (0 ==
                  pthread_mutex_init (&sem->mutex,
                                      NULL));
   GNUNET_assert (0 ==
                  pthread_cond_init (&sem->cv,
                                     NULL));
   sem->ctr = val;
 }
 
 
 /**
  * Decrement semaphore, blocks until this is possible.
  *
  * @param[in,out] sem semaphore to decrement
  */
 static void
 sem_down (struct Semaphore *sem)
 {
   GNUNET_assert (0 == pthread_mutex_lock (&sem->mutex));
   while (0 == sem->ctr)
   {
     pthread_cond_wait (&sem->cv,
                        &sem->mutex);
   }
   sem->ctr--;
   GNUNET_assert (0 == pthread_mutex_unlock (&sem->mutex));
 }
 
 
 /**
  * Increment semaphore, blocks until this is possible.
  *
  * @param[in,out] sem semaphore to decrement
  */
 static void
 sem_up (struct Semaphore *sem)
 {
   GNUNET_assert (0 == pthread_mutex_lock (&sem->mutex));
   sem->ctr++;
   GNUNET_assert (0 == pthread_mutex_unlock (&sem->mutex));
   pthread_cond_signal (&sem->cv);
 }
 
 
 /**
  * Release resources used by @a sem.
  *
  * @param[in] sem semaphore to release (except the memory itself)
  */
 static void
 sem_done (struct Semaphore *sem)
 {
   GNUNET_break (0 == pthread_cond_destroy (&sem->cv));
   GNUNET_break (0 == pthread_mutex_destroy (&sem->mutex));
 }
 
 
 /**
  * Main logic of a worker thread. Grabs work, does it,
  * grabs more work.
  *
  * @param cls a `struct Worker *`
  * @returns cls
  */
 static void *
 worker (void *cls)
 {
   struct Worker *w = cls;
 
   while (true)
   {
     GNUNET_assert (0 == pthread_mutex_lock (&worker_lock));
     GNUNET_CONTAINER_DLL_insert (worker_head,
                                  worker_tail,
                                  w);
     GNUNET_assert (0 == pthread_mutex_unlock (&worker_lock));
     sem_up (&worker_sem);
     sem_down (&w->sem);
     if (w->do_shutdown)
       break;
     {
       struct BatchJob *bj = w->job;
 
       switch (bj->type)
       {
       case TYPE_SIGN:
         {
           const struct TALER_CRYPTO_CsSignRequestMessage *sr
             = bj->details.sign.sr;
 
           bj->ec = do_sign (&sr->h_cs,
                             &sr->message,
                             (0 != ntohl (sr->for_melt)),
                             &bj->details.sign.cs_answer);
           break;
         }
       case TYPE_RDERIVE:
         {
           const struct TALER_CRYPTO_CsRDeriveRequest *rdr
             = bj->details.rderive.rdr;
           bj->ec = do_derive (&rdr->h_cs,
                               &rdr->nonce,
                               (0 != ntohl (rdr->for_melt)),
                               &bj->details.rderive.rpairp);
           break;
         }
       }
       sem_up (&bj->sem);
       w->job = NULL;
     }
   }
   return w;
 }
 
 
 /**
  * Start batch job @a bj to sign @a sr.
  *
  * @param sr signature request to answer
  * @param[out] bj job data structure
  */
 static void
 start_sign_job (const struct TALER_CRYPTO_CsSignRequestMessage *sr,
                 struct BatchJob *bj)
 {
   sem_init (&bj->sem,
             0);
   bj->type = TYPE_SIGN;
   bj->details.sign.sr = sr;
   sem_down (&worker_sem);
   GNUNET_assert (0 == pthread_mutex_lock (&worker_lock));
   bj->worker = worker_head;
   GNUNET_CONTAINER_DLL_remove (worker_head,
                                worker_tail,
                                bj->worker);
   GNUNET_assert (0 == pthread_mutex_unlock (&worker_lock));
   bj->worker->job = bj;
   sem_up (&bj->worker->sem);
 }
 
 
 /**
  * Start batch job @a bj to derive @a rdr.
  *
  * @param rdr derivation request to answer
  * @param[out] bj job data structure
  */
 static void
 start_derive_job (const struct TALER_CRYPTO_CsRDeriveRequest *rdr,
                   struct BatchJob *bj)
 {
   sem_init (&bj->sem,
             0);
   bj->type = TYPE_RDERIVE;
   bj->details.rderive.rdr = rdr;
   sem_down (&worker_sem);
   GNUNET_assert (0 == pthread_mutex_lock (&worker_lock));
   bj->worker = worker_head;
   GNUNET_CONTAINER_DLL_remove (worker_head,
                                worker_tail,
                                bj->worker);
   GNUNET_assert (0 == pthread_mutex_unlock (&worker_lock));
   bj->worker->job = bj;
   sem_up (&bj->worker->sem);
 }
 
 
 /**
  * Finish a job @a bj for a @a client.
  *
  * @param client who made the request
  * @param[in,out] bj job to finish
  */
 static void
 finish_job (struct TES_Client *client,
             struct BatchJob *bj)
 {
   sem_down (&bj->sem);
   sem_done (&bj->sem);
   switch (bj->type)
   {
   case TYPE_SIGN:
     if (TALER_EC_NONE != bj->ec)
     {
       fail_sign (client,
                  bj->ec);
       return;
     }
     send_signature (client,
                     &bj->details.sign.cs_answer);
     break;
   case TYPE_RDERIVE:
     if (TALER_EC_NONE != bj->ec)
     {
       fail_derive (client,
                    bj->ec);
       return;
     }
     send_derivation (client,
                      &bj->details.rderive.rpairp);
     break;
   }
 }
 
 
 /**
  * Handle @a client request @a sr to create a batch of signature. Creates the
  * signatures using the respective key and return the results to the client.
  *
  * @param client the client making the request
  * @param bsr the request details
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 handle_batch_sign_request (struct TES_Client *client,
                            const struct TALER_CRYPTO_BatchSignRequest *bsr)
 {
   uint32_t bs = ntohl (bsr->batch_size);
   uint16_t size = ntohs (bsr->header.size) - sizeof (*bsr);
   const void *off = (const void *) &bsr[1];
   unsigned int idx = 0;
   bool failure = false;
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Handling batch sign request of size %u\n",
               (unsigned int) bs);
   if (bs > TALER_MAX_COINS)
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
   {
     struct BatchJob jobs[GNUNET_NZL (bs)];
 
     while ( (idx < bs) &&
             (size >= sizeof (struct TALER_CRYPTO_CsSignRequestMessage)) )
     {
       const struct TALER_CRYPTO_CsSignRequestMessage *sr = off;
       uint16_t s = ntohs (sr->header.size);
 
       if (s != sizeof (*sr))
       {
         failure = true;
         bs = idx;
         break;
       }
       start_sign_job (sr,
                       &jobs[idx++]);
       off += s;
       size -= s;
     }
     GNUNET_break_op (0 == size);
     bs = GNUNET_MIN (bs,
                      idx);
     for (unsigned int i = 0; i<bs; i++)
       finish_job (client,
                   &jobs[i]);
   }
   if (failure)
   {
     struct TALER_CRYPTO_SignFailure sf = {
       .header.size = htons (sizeof (sf)),
       .header.type = htons (TALER_HELPER_CS_MT_RES_BATCH_SIGN_FAILURE),
       .ec = htonl (TALER_EC_GENERIC_INTERNAL_INVARIANT_FAILURE)
     };
 
     GNUNET_break (0);
     return TES_transmit (client->csock,
                          &sf.header);
   }
   return GNUNET_OK;
 }
 
 
 /**
  * Handle @a client request @a sr to create a batch of derivations. Creates the
  * derivations using the respective key and return the results to the client.
  *
  * @param client the client making the request
  * @param bdr the request details
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 handle_batch_derive_request (struct TES_Client *client,
                              const struct TALER_CRYPTO_BatchDeriveRequest *bdr)
 {
   uint32_t bs = ntohl (bdr->batch_size);
   uint16_t size = ntohs (bdr->header.size) - sizeof (*bdr);
   const void *off = (const void *) &bdr[1];
   unsigned int idx = 0;
   bool failure = false;
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Handling batch derivation request of size %u\n",
               (unsigned int) bs);
   if (bs > TALER_MAX_COINS)
   {
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
   {
     struct BatchJob jobs[GNUNET_NZL (bs)];
 
     while ( (idx < bs) &&
             (size >= sizeof (struct TALER_CRYPTO_CsRDeriveRequest)) )
     {
       const struct TALER_CRYPTO_CsRDeriveRequest *rdr = off;
       uint16_t s = ntohs (rdr->header.size);
 
       if ( (s > size) ||
            (s != sizeof (*rdr)) )
       {
         failure = true;
         bs = idx;
         break;
       }
       start_derive_job (rdr,
                         &jobs[idx++]);
       off += s;
       size -= s;
     }
     GNUNET_break_op (0 == size);
     bs = GNUNET_MIN (bs,
                      idx);
     for (unsigned int i = 0; i<bs; i++)
       finish_job (client,
                   &jobs[i]);
   }
   if (failure)
   {
     GNUNET_break (0);
     return fail_derive (client,
                         TALER_EC_GENERIC_INTERNAL_INVARIANT_FAILURE);
   }
   return GNUNET_OK;
 }
 
 
 /**
  * Start worker thread for batch processing.
  *
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 start_worker (void)
 {
   struct Worker *w;
 
   w = GNUNET_new (struct Worker);
   sem_init (&w->sem,
             0);
   if (0 != pthread_create (&w->pt,
                            NULL,
                            &worker,
                            w))
   {
     GNUNET_log_strerror (GNUNET_ERROR_TYPE_ERROR,
                          "pthread_create");
     GNUNET_free (w);
     return GNUNET_SYSERR;
   }
   workers++;
   return GNUNET_OK;
 }
 
 
 /**
  * Stop all worker threads.
  */
 static void
 stop_workers (void)
 {
   while (workers > 0)
   {
     struct Worker *w;
     void *result;
 
     sem_down (&worker_sem);
     GNUNET_assert (0 == pthread_mutex_lock (&worker_lock));
     w = worker_head;
     GNUNET_CONTAINER_DLL_remove (worker_head,
                                  worker_tail,
                                  w);
     GNUNET_assert (0 == pthread_mutex_unlock (&worker_lock));
     w->do_shutdown = true;
     sem_up (&w->sem);
     pthread_join (w->pt,
                   &result);
     GNUNET_assert (result == w);
     sem_done (&w->sem);
     GNUNET_free (w);
     workers--;
   }
 }
 
 
 /**
  * Initialize key material for denomination key @a dk (also on disk).
  *
  * @param[in,out] dk denomination key to compute key material for
  * @param position where in the DLL will the @a dk go
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 setup_key (struct DenominationKey *dk,
            struct DenominationKey *position)
 {
   struct Denomination *denom = dk->denom;
   struct GNUNET_CRYPTO_CsPrivateKey priv;
   struct GNUNET_CRYPTO_CsPublicKey pub;
 
   GNUNET_CRYPTO_cs_private_key_generate (&priv);
   GNUNET_CRYPTO_cs_private_key_get_public (&priv,
                                            &pub);
   GNUNET_CRYPTO_hash (&pub,
                       sizeof (pub),
                       &dk->h_cs.hash);
   GNUNET_asprintf (
     &dk->filename,
     "%s/%s/%llu-%llu",
     keydir,
     denom->section,
     (unsigned long long) (dk->anchor_start.abs_time.abs_value_us
                           / GNUNET_TIME_UNIT_SECONDS.rel_value_us
                           ),
     (unsigned long long) (dk->anchor_end.abs_time.abs_value_us
                           / GNUNET_TIME_UNIT_SECONDS.rel_value_us
                           ));
   if (GNUNET_OK !=
       GNUNET_DISK_fn_write (dk->filename,
                             &priv,
                             sizeof(priv),
                             GNUNET_DISK_PERM_USER_READ))
   {
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                               "write",
                               dk->filename);
     return GNUNET_SYSERR;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Setup fresh private key %s at %s in `%s' (generation #%llu)\n",
               GNUNET_h2s (&dk->h_cs.hash),
               GNUNET_TIME_timestamp2s (dk->anchor_start),
               dk->filename,
               (unsigned long long) key_gen);
   dk->denom_priv = priv;
   dk->denom_pub = pub;
   dk->key_gen = key_gen;
   generate_response (dk);
   if (GNUNET_OK !=
       GNUNET_CONTAINER_multihashmap_put (
         keys,
         &dk->h_cs.hash,
         dk,
         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "Duplicate private key created! Terminating.\n");
     GNUNET_free (dk->filename);
     GNUNET_free (dk->an);
     GNUNET_free (dk);
     return GNUNET_SYSERR;
   }
   GNUNET_CONTAINER_DLL_insert_after (denom->keys_head,
                                      denom->keys_tail,
                                      position,
                                      dk);
   return GNUNET_OK;
 }
 
 
 /**
  * The withdraw period of a key @a dk has expired. Purge it.
  *
  * @param[in] dk expired denomination key to purge
  */
 static void
 purge_key (struct DenominationKey *dk)
 {
   if (dk->purge)
     return;
   if (0 != unlink (dk->filename))
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                               "unlink",
                               dk->filename);
   GNUNET_free (dk->filename);
   dk->purge = true;
   dk->key_gen = key_gen;
 }
 
 
 /**
  * A @a client informs us that a key has been revoked.
  * Check if the key is still in use, and if so replace (!)
  * it with a fresh key.
  *
  * @param client the client making the request
  * @param rr the revocation request
  */
 static enum GNUNET_GenericReturnValue
 handle_revoke_request (struct TES_Client *client,
                        const struct TALER_CRYPTO_CsRevokeRequest *rr)
 {
   struct DenominationKey *dk;
   struct DenominationKey *ndk;
   struct Denomination *denom;
 
   (void) client;
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   dk = GNUNET_CONTAINER_multihashmap_get (keys,
                                           &rr->h_cs.hash);
   if (NULL == dk)
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "Revocation request ignored, denomination key %s unknown\n",
                 GNUNET_h2s (&rr->h_cs.hash));
     return GNUNET_OK;
   }
   if (dk->purge)
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "Revocation request ignored, denomination key %s already revoked\n",
                 GNUNET_h2s (&rr->h_cs.hash));
     return GNUNET_OK;
   }
 
   key_gen++;
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Revoking key %s, bumping generation to %llu\n",
               GNUNET_h2s (&rr->h_cs.hash),
               (unsigned long long) key_gen);
   purge_key (dk);
 
   /* Setup replacement key */
   denom = dk->denom;
   ndk = GNUNET_new (struct DenominationKey);
   ndk->denom = denom;
   ndk->anchor_start = dk->anchor_start;
   ndk->anchor_end = dk->anchor_end;
   if (GNUNET_OK !=
       setup_key (ndk,
                  dk))
   {
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     GNUNET_break (0);
     GNUNET_SCHEDULER_shutdown ();
     globals->global_ret = EXIT_FAILURE;
     return GNUNET_SYSERR;
   }
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   TES_wake_clients ();
   return GNUNET_OK;
 }
 
 
 /**
  * Handle @a client request @a rdr to create signature. Create the
  * signature using the respective key and return the result to
  * the client.
  *
  * @param client the client making the request
  * @param rdr the request details
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 handle_r_derive_request (struct TES_Client *client,
                          const struct TALER_CRYPTO_CsRDeriveRequest *rdr)
 {
   struct GNUNET_CRYPTO_CSPublicRPairP r_pub;
   struct GNUNET_TIME_Absolute now = GNUNET_TIME_absolute_get ();
   enum TALER_ErrorCode ec;
   enum GNUNET_GenericReturnValue ret;
 
   ec = do_derive (&rdr->h_cs,
                   &rdr->nonce,
                   (0 != ntohl (rdr->for_melt)),
                   &r_pub);
   if (TALER_EC_NONE != ec)
   {
     return fail_derive (client,
                         ec);
   }
 
   ret = send_derivation (client,
                          &r_pub);
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Sent CS Derived R after %s\n",
               GNUNET_TIME_relative2s (
                 GNUNET_TIME_absolute_get_duration (now),
                 GNUNET_YES));
   return ret;
 }
 
 
 /**
  * Handle @a hdr message received from @a client.
  *
  * @param client the client that received the message
  * @param hdr message that was received
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 cs_work_dispatch (struct TES_Client *client,
                   const struct GNUNET_MessageHeader *hdr)
 {
   uint16_t msize = ntohs (hdr->size);
 
   switch (ntohs (hdr->type))
   {
   case TALER_HELPER_CS_MT_REQ_SIGN:
     if (msize < sizeof (struct TALER_CRYPTO_CsSignRequestMessage))
     {
       GNUNET_break_op (0);
       return GNUNET_SYSERR;
     }
     return handle_sign_request (
       client,
       (const struct TALER_CRYPTO_CsSignRequestMessage *) hdr);
   case TALER_HELPER_CS_MT_REQ_REVOKE:
     if (msize != sizeof (struct TALER_CRYPTO_CsRevokeRequest))
     {
       GNUNET_break_op (0);
       return GNUNET_SYSERR;
     }
     return handle_revoke_request (
       client,
       (const struct TALER_CRYPTO_CsRevokeRequest *) hdr);
   case TALER_HELPER_CS_MT_REQ_BATCH_SIGN:
     if (msize <= sizeof (struct TALER_CRYPTO_BatchSignRequest))
     {
       GNUNET_break_op (0);
       return GNUNET_SYSERR;
     }
     return handle_batch_sign_request (
       client,
       (const struct TALER_CRYPTO_BatchSignRequest *) hdr);
   case TALER_HELPER_CS_MT_REQ_BATCH_RDERIVE:
     if (msize <= sizeof (struct TALER_CRYPTO_BatchDeriveRequest))
     {
       GNUNET_break_op (0);
       return GNUNET_SYSERR;
     }
     return handle_batch_derive_request (
       client,
       (const struct TALER_CRYPTO_BatchDeriveRequest *) hdr);
   case TALER_HELPER_CS_MT_REQ_RDERIVE:
     if (msize != sizeof (struct TALER_CRYPTO_CsRDeriveRequest))
     {
       GNUNET_break_op (0);
       return GNUNET_SYSERR;
     }
     return handle_r_derive_request (client,
                                     (const struct
                                      TALER_CRYPTO_CsRDeriveRequest *) hdr);
   default:
     GNUNET_break_op (0);
     return GNUNET_SYSERR;
   }
 }
 
 
 /**
  * Send our initial key set to @a client together with the
  * "sync" terminator.
  *
  * @param client the client to inform
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 cs_client_init (struct TES_Client *client)
 {
   size_t obs = 0;
   char *buf;
 
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Initializing new client %p\n",
               client);
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   for (struct Denomination *denom = denom_head;
        NULL != denom;
        denom = denom->next)
   {
     for (struct DenominationKey *dk = denom->keys_head;
          NULL != dk;
          dk = dk->next)
     {
       obs += ntohs (dk->an->header.size);
     }
   }
   buf = GNUNET_malloc (obs);
   obs = 0;
   for (struct Denomination *denom = denom_head;
        NULL != denom;
        denom = denom->next)
   {
     for (struct DenominationKey *dk = denom->keys_head;
          NULL != dk;
          dk = dk->next)
     {
       GNUNET_memcpy (&buf[obs],
                      dk->an,
                      ntohs (dk->an->header.size));
       obs += ntohs (dk->an->header.size);
     }
   }
   client->key_gen = key_gen;
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   if (GNUNET_OK !=
       TES_transmit_raw (client->csock,
                         obs,
                         buf))
   {
     GNUNET_free (buf);
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Client %p must have disconnected\n",
                 client);
     return GNUNET_SYSERR;
   }
   GNUNET_free (buf);
   {
     struct GNUNET_MessageHeader synced = {
       .type = htons (TALER_HELPER_CS_SYNCED),
       .size = htons (sizeof (synced))
     };
 
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Sending CS SYNCED message to %p\n",
                 client);
     if (GNUNET_OK !=
         TES_transmit (client->csock,
                       &synced))
     {
       GNUNET_break (0);
       return GNUNET_SYSERR;
     }
   }
   return GNUNET_OK;
 }
 
 
 /**
  * Notify @a client about all changes to the keys since
  * the last generation known to the @a client.
  *
  * @param client the client to notify
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 cs_update_client_keys (struct TES_Client *client)
 {
   size_t obs = 0;
   char *buf;
   enum GNUNET_GenericReturnValue ret;
 
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   for (struct Denomination *denom = denom_head;
        NULL != denom;
        denom = denom->next)
   {
     for (struct DenominationKey *key = denom->keys_head;
          NULL != key;
          key = key->next)
     {
       if (key->key_gen <= client->key_gen)
         continue;
       if (key->purge)
         obs += sizeof (struct TALER_CRYPTO_CsKeyPurgeNotification);
       else
         obs += ntohs (key->an->header.size);
     }
   }
   if (0 == obs)
   {
     /* nothing to do */
     client->key_gen = key_gen;
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     return GNUNET_OK;
   }
   buf = GNUNET_malloc (obs);
   obs = 0;
   for (struct Denomination *denom = denom_head;
        NULL != denom;
        denom = denom->next)
   {
     for (struct DenominationKey *key = denom->keys_head;
          NULL != key;
          key = key->next)
     {
       if (key->key_gen <= client->key_gen)
         continue;
       if (key->purge)
       {
         struct TALER_CRYPTO_CsKeyPurgeNotification pn = {
           .header.type = htons (TALER_HELPER_CS_MT_PURGE),
           .header.size = htons (sizeof (pn)),
           .h_cs = key->h_cs
         };
 
         GNUNET_memcpy (&buf[obs],
                        &pn,
                        sizeof (pn));
         GNUNET_assert (obs + sizeof (pn)
                        > obs);
         obs += sizeof (pn);
       }
       else
       {
         GNUNET_memcpy (&buf[obs],
                        key->an,
                        ntohs (key->an->header.size));
         GNUNET_assert (obs + ntohs (key->an->header.size)
                        > obs);
         obs += ntohs (key->an->header.size);
       }
     }
   }
   client->key_gen = key_gen;
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   ret = TES_transmit_raw (client->csock,
                           obs,
                           buf);
   GNUNET_free (buf);
   return ret;
 }
 
 
 /**
  * Create a new denomination key (we do not have enough).
  *
  * @param[in,out] denom denomination key to create
  * @param anchor_start when to start key signing validity
  * @param anchor_end when to end key signing validity
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 create_key (struct Denomination *denom,
             struct GNUNET_TIME_Timestamp anchor_start,
             struct GNUNET_TIME_Timestamp anchor_end)
 {
   struct DenominationKey *dk;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Creating new key for `%s' with start date %s\n",
               denom->section,
               GNUNET_TIME_timestamp2s (anchor_start));
   dk = GNUNET_new (struct DenominationKey);
   dk->denom = denom;
   dk->anchor_start = anchor_start;
   dk->anchor_end = anchor_end;
   if (GNUNET_OK !=
       setup_key (dk,
                  denom->keys_tail))
   {
     GNUNET_break (0);
     GNUNET_free (dk);
     GNUNET_SCHEDULER_shutdown ();
     globals->global_ret = EXIT_FAILURE;
     return GNUNET_SYSERR;
   }
   return GNUNET_OK;
 }
 
 
 /**
  * Obtain the maximum withdraw duration of all denominations.
  *
  * Must only be called while the #keys_lock is held.
  *
  * @return maximum withdraw duration, zero if there are no denominations
  */
 static struct GNUNET_TIME_Relative
 get_maximum_duration (void)
 {
   struct GNUNET_TIME_Relative ret
     = GNUNET_TIME_UNIT_ZERO;
 
   for (struct Denomination *denom = denom_head;
        NULL != denom;
        denom = denom->next)
   {
     ret = GNUNET_TIME_relative_max (ret,
                                     denom->duration_withdraw);
   }
   return ret;
 }
 
 
 /**
  * At what time do we need to next create keys if we just did?
  *
  * @return time when to next create keys if we just finished key generation
  */
 static struct GNUNET_TIME_Absolute
 action_time (void)
 {
   struct GNUNET_TIME_Relative md = get_maximum_duration ();
   struct GNUNET_TIME_Absolute now = GNUNET_TIME_absolute_get ();
   uint64_t mod;
 
   if (GNUNET_TIME_relative_is_zero (md))
     return GNUNET_TIME_UNIT_FOREVER_ABS;
   mod = now.abs_value_us % md.rel_value_us;
   now.abs_value_us -= mod;
   return GNUNET_TIME_absolute_add (now,
                                    md);
 }
 
 
 /**
  * Remove all denomination keys of @a denom that have expired.
  *
  * @param[in,out] denom denomination family to remove keys for
  */
 static void
 remove_expired_denomination_keys (struct Denomination *denom)
 {
   while ( (NULL != denom->keys_head) &&
           GNUNET_TIME_absolute_is_past (
             denom->keys_head->anchor_end.abs_time) )
   {
     struct DenominationKey *key = denom->keys_head;
     struct DenominationKey *nxt = key->next;
 
     if (0 != key->rc)
       break; /* later */
     GNUNET_CONTAINER_DLL_remove (denom->keys_head,
                                  denom->keys_tail,
                                  key);
     GNUNET_assert (GNUNET_OK ==
                    GNUNET_CONTAINER_multihashmap_remove (
                      keys,
                      &key->h_cs.hash,
                      key));
     if ( (! key->purge) &&
          (0 != unlink (key->filename)) )
       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
                                 "unlink",
                                 key->filename);
     GNUNET_free (key->filename);
     GNUNET_free (key->an);
     GNUNET_free (key);
     key = nxt;
   }
 }
 
 
 /**
  * Obtain the end anchor to use at this point. Uses the
  * #lookahead_sign and then rounds it up by the maximum
  * duration of any denomination to arrive at a globally
  * valid end-date.
  *
  * Must only be called while the #keys_lock is held.
  *
  * @return end anchor
  */
 static struct GNUNET_TIME_Timestamp
 get_anchor_end (void)
 {
   struct GNUNET_TIME_Relative md = get_maximum_duration ();
   struct GNUNET_TIME_Absolute end
     = GNUNET_TIME_relative_to_absolute (lookahead_sign);
   uint64_t mod;
 
   if (GNUNET_TIME_relative_is_zero (md))
     return GNUNET_TIME_UNIT_ZERO_TS;
   /* Round up 'end' to a multiple of 'md' */
   mod = end.abs_value_us % md.rel_value_us;
   end.abs_value_us -= mod;
   return GNUNET_TIME_absolute_to_timestamp (
     GNUNET_TIME_absolute_add (end,
                               md));
 }
 
 
 /**
  * Create all denomination keys that are required for our
  * desired lookahead and that we do not yet have.
  *
  * @param[in,out] opt our options
  * @param[in,out] wake set to true if we should wake the clients
  */
 static void
 create_missing_keys (struct TALER_SECMOD_Options *opt,
                      bool *wake)
 {
   struct GNUNET_TIME_Timestamp start;
   struct GNUNET_TIME_Timestamp end;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Updating denominations ...\n");
   start = opt->global_now;
   GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
   end = get_anchor_end ();
   for (struct Denomination *denom = denom_head;
        NULL != denom;
        denom = denom->next)
   {
     struct GNUNET_TIME_Timestamp anchor_start;
     struct GNUNET_TIME_Timestamp anchor_end;
     struct GNUNET_TIME_Timestamp next_end;
     bool finished = false;
 
     remove_expired_denomination_keys (denom);
     if (NULL != denom->keys_tail)
     {
       anchor_start = denom->keys_tail->anchor_end;
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                   "Expanding keys of denomination `%s', last key %s valid for another %s\n",
                   denom->section,
                   GNUNET_h2s (&denom->keys_tail->h_cs.hash),
                   GNUNET_TIME_relative2s (
                     GNUNET_TIME_absolute_get_remaining (
                       anchor_start.abs_time),
                     true));
     }
     else
     {
       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                   "Starting keys of denomination `%s'\n",
                   denom->section);
       anchor_start = start;
     }
     finished = GNUNET_TIME_timestamp_cmp (anchor_start,
                                           >=,
                                           end);
     while (! finished)
     {
       anchor_end = GNUNET_TIME_absolute_to_timestamp (
         GNUNET_TIME_absolute_add (anchor_start.abs_time,
                                   denom->duration_withdraw));
       next_end = GNUNET_TIME_absolute_to_timestamp (
         GNUNET_TIME_absolute_add (anchor_end.abs_time,
                                   denom->duration_withdraw));
       if (GNUNET_TIME_timestamp_cmp (next_end,
                                      >,
                                      end))
       {
         anchor_end = end; /* extend period to align end periods */
         finished = true;
       }
       /* adjust start time down to ensure overlap */
       anchor_start = GNUNET_TIME_absolute_to_timestamp (
         GNUNET_TIME_absolute_subtract (anchor_start.abs_time,
                                        overlap_duration));
       if (! *wake)
       {
         key_gen++;
         *wake = true;
       }
       if (GNUNET_OK !=
           create_key (denom,
                       anchor_start,
                       anchor_end))
       {
         GNUNET_break (0);
         GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
         globals->global_ret = EXIT_FAILURE;
         GNUNET_SCHEDULER_shutdown ();
         return;
       }
       anchor_start = anchor_end;
     }
     remove_expired_denomination_keys (denom);
   }
   GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Updating denominations finished ...\n");
 }
 
 
 /**
  * Task run periodically to expire keys and/or generate fresh ones.
  *
  * @param cls the `struct TALER_SECMOD_Options *`
  */
 static void
 update_denominations (void *cls)
 {
   struct TALER_SECMOD_Options *opt = cls;
   struct GNUNET_TIME_Absolute at;
   bool wake = false;
 
   (void) cls;
   keygen_task = NULL;
   opt->global_now = GNUNET_TIME_timestamp_get ();
   create_missing_keys (opt,
                        &wake);
   if (wake)
     TES_wake_clients ();
   at = action_time ();
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Next key generation due at %s\n",
               GNUNET_TIME_absolute2s (at));
   keygen_task = GNUNET_SCHEDULER_add_at (at,
                                          &update_denominations,
                                          opt);
 }
 
 
 /**
  * Parse private key of denomination @a denom in @a buf.
  *
  * @param[out] denom denomination of the key
  * @param filename name of the file we are parsing, for logging
  * @param priv key material
  */
 static void
 parse_key (struct Denomination *denom,
            const char *filename,
            const struct GNUNET_CRYPTO_CsPrivateKey *priv)
 {
   const char *anchor_s;
   char dummy;
   unsigned long long anchor_start_ll;
   unsigned long long anchor_end_ll;
   struct GNUNET_TIME_Timestamp anchor_start;
   struct GNUNET_TIME_Timestamp anchor_end;
   char *nf = NULL;
 
   anchor_s = strrchr (filename,
                       '/');
   if (NULL == anchor_s)
   {
     /* File in a directory without '/' in the name, this makes no sense. */
     GNUNET_break (0);
     return;
   }
   anchor_s++;
   if (2 != sscanf (anchor_s,
                    "%llu-%llu%c",
                    &anchor_start_ll,
                    &anchor_end_ll,
                    &dummy))
   {
     /* try legacy mode */
     if (1 != sscanf (anchor_s,
                      "%llu%c",
                      &anchor_start_ll,
                      &dummy))
     {
       /* Filenames in KEYDIR must ONLY be the anchor time in seconds! */
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                   "Filename `%s' invalid for key file, skipping\n",
                   anchor_s);
       return;
     }
     anchor_start.abs_time.abs_value_us
       = anchor_start_ll * GNUNET_TIME_UNIT_SECONDS.rel_value_us;
     if (anchor_start_ll != anchor_start.abs_time.abs_value_us
         / GNUNET_TIME_UNIT_SECONDS.rel_value_us)
     {
       /* Integer overflow. Bad, invalid filename. */
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                   "Integer overflow. Filename `%s' invalid for key file, skipping\n",
                   anchor_s);
       return;
     }
     anchor_end
       = GNUNET_TIME_absolute_to_timestamp (
           GNUNET_TIME_absolute_add (anchor_start.abs_time,
                                     denom->duration_withdraw));
     GNUNET_asprintf (
       &nf,
       "%s/%s/%llu-%llu",
       keydir,
       denom->section,
       anchor_start_ll,
       (unsigned long long) (anchor_end.abs_time.abs_value_us
                             / GNUNET_TIME_UNIT_SECONDS.rel_value_us));
     /* Try to fix the legacy filename */
     if (0 !=
         rename (filename,
                 nf))
     {
       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                 "rename",
                                 filename);
       GNUNET_free (nf);
     }
   }
   else
   {
     anchor_start.abs_time.abs_value_us
       = anchor_start_ll * GNUNET_TIME_UNIT_SECONDS.rel_value_us;
     anchor_end.abs_time.abs_value_us
       = anchor_end_ll * GNUNET_TIME_UNIT_SECONDS.rel_value_us;
     if ( (anchor_start_ll != anchor_start.abs_time.abs_value_us
           / GNUNET_TIME_UNIT_SECONDS.rel_value_us) ||
          (anchor_end_ll != anchor_end.abs_time.abs_value_us
           / GNUNET_TIME_UNIT_SECONDS.rel_value_us) )
     {
       /* Integer overflow. Bad, invalid filename. */
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                   "Integer overflow. Filename `%s' invalid for key file, skipping\n",
                   anchor_s);
       return;
     }
   }
 
   {
     struct DenominationKey *dk;
     struct DenominationKey *before;
 
     dk = GNUNET_new (struct DenominationKey);
     dk->denom_priv = *priv;
     dk->denom = denom;
     dk->anchor_start = anchor_start;
     dk->anchor_end = anchor_end;
     dk->filename = (NULL == nf) ? GNUNET_strdup (filename) : nf;
     GNUNET_CRYPTO_cs_private_key_get_public (priv,
                                              &dk->denom_pub);
     GNUNET_CRYPTO_hash (&dk->denom_pub,
                         sizeof (dk->denom_pub),
                         &dk->h_cs.hash);
     generate_response (dk);
     if (GNUNET_OK !=
         GNUNET_CONTAINER_multihashmap_put (
           keys,
           &dk->h_cs.hash,
           dk,
           GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "Duplicate private key %s detected in file `%s'. Skipping.\n",
                   GNUNET_h2s (&dk->h_cs.hash),
                   filename);
       GNUNET_free (dk->an);
       GNUNET_free (dk);
       return;
     }
     before = NULL;
     for (struct DenominationKey *pos = denom->keys_head;
          NULL != pos;
          pos = pos->next)
     {
       if (GNUNET_TIME_timestamp_cmp (pos->anchor_start,
                                      >,
                                      anchor_start))
         break;
       before = pos;
     }
     GNUNET_CONTAINER_DLL_insert_after (denom->keys_head,
                                        denom->keys_tail,
                                        before,
                                        dk);
     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
                 "Imported key %s from `%s'\n",
                 GNUNET_h2s (&dk->h_cs.hash),
                 filename);
   }
 }
 
 
 /**
  * Import a private key from @a filename for the denomination
  * given in @a cls.
  *
  * @param[in,out] cls a `struct Denomiantion`
  * @param filename name of a file in the directory
  * @return #GNUNET_OK (always, continue to iterate)
  */
 static enum GNUNET_GenericReturnValue
 import_key (void *cls,
             const char *filename)
 {
   struct Denomination *denom = cls;
   struct GNUNET_DISK_FileHandle *fh;
   struct GNUNET_DISK_MapHandle *map;
   void *ptr;
   int fd;
   struct stat sbuf;
 
   {
     struct stat lsbuf;
 
     if (0 != lstat (filename,
                     &lsbuf))
     {
       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                 "lstat",
                                 filename);
       return GNUNET_OK;
     }
     if (! S_ISREG (lsbuf.st_mode))
     {
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                   "File `%s' is not a regular file, which is not allowed for private keys!\n",
                   filename);
       return GNUNET_OK;
     }
   }
 
   fd = open (filename,
              O_RDONLY | O_CLOEXEC);
   if (-1 == fd)
   {
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                               "open",
                               filename);
     return GNUNET_OK;
   }
   if (0 != fstat (fd,
                   &sbuf))
   {
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                               "stat",
                               filename);
     GNUNET_break (0 == close (fd));
     return GNUNET_OK;
   }
   if (! S_ISREG (sbuf.st_mode))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "File `%s' is not a regular file, which is not allowed for private keys!\n",
                 filename);
     GNUNET_break (0 == close (fd));
     return GNUNET_OK;
   }
   if (0 != (sbuf.st_mode & (S_IWUSR | S_IRWXG | S_IRWXO)))
   {
     /* permission are NOT tight, try to patch them up! */
     if (0 !=
         fchmod (fd,
                 S_IRUSR))
     {
       GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                                 "fchmod",
                                 filename);
       /* refuse to use key if file has wrong permissions */
       GNUNET_break (0 == close (fd));
       return GNUNET_OK;
     }
   }
   fh = GNUNET_DISK_get_handle_from_int_fd (fd);
   if (NULL == fh)
   {
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                               "open",
                               filename);
     GNUNET_break (0 == close (fd));
     return GNUNET_OK;
   }
   if (sbuf.st_size != sizeof(struct GNUNET_CRYPTO_CsPrivateKey))
   {
     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
                 "File `%s' too big to be a private key\n",
                 filename);
     GNUNET_DISK_file_close (fh);
     return GNUNET_OK;
   }
   ptr = GNUNET_DISK_file_map (fh,
                               &map,
                               GNUNET_DISK_MAP_TYPE_READ,
                               (size_t) sbuf.st_size);
   if (NULL == ptr)
   {
     GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING,
                               "mmap",
                               filename);
     GNUNET_DISK_file_close (fh);
     return GNUNET_OK;
   }
   parse_key (denom,
              filename,
              (const struct GNUNET_CRYPTO_CsPrivateKey *) ptr);
   GNUNET_DISK_file_unmap (map);
   GNUNET_DISK_file_close (fh);
   return GNUNET_OK;
 }
 
 
 /**
  * Parse configuration for denomination type parameters.  Also determines
  * our anchor by looking at the existing denominations of the same type.
  *
  * @param cfg configuration to use
  * @param ct section in the configuration file giving the denomination type parameters
  * @param[out] denom set to the denomination parameters from the configuration
  * @return #GNUNET_OK on success, #GNUNET_SYSERR if the configuration is invalid
  */
 static enum GNUNET_GenericReturnValue
 parse_denomination_cfg (const struct GNUNET_CONFIGURATION_Handle *cfg,
                         const char *ct,
                         struct Denomination *denom)
 {
   char *secname;
 
   GNUNET_asprintf (&secname,
                    "%s-secmod-cs",
                    globals->section);
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_time (cfg,
                                            ct,
                                            "DURATION_WITHDRAW",
                                            &denom->duration_withdraw))
   {
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                                ct,
                                "DURATION_WITHDRAW");
     GNUNET_free (secname);
     return GNUNET_SYSERR;
   }
   if (GNUNET_TIME_relative_cmp (denom->duration_withdraw,
                                 <,
                                 GNUNET_TIME_UNIT_SECONDS))
   {
     GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
                                ct,
                                "DURATION_WITHDRAW",
                                "less than one second is not supported");
     GNUNET_free (secname);
     return GNUNET_SYSERR;
   }
   if (GNUNET_TIME_relative_cmp (overlap_duration,
                                 >=,
                                 denom->duration_withdraw))
   {
     GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
                                secname,
                                "OVERLAP_DURATION",
                                "Value given must be smaller than value for DURATION_WITHDRAW!");
     GNUNET_free (secname);
     return GNUNET_SYSERR;
   }
   GNUNET_free (secname);
   denom->section = GNUNET_strdup (ct);
   return GNUNET_OK;
 }
 
 
 /**
  * Closure for #load_denominations.
  */
 struct LoadContext
 {
 
   /**
    * Configuration to use.
    */
   const struct GNUNET_CONFIGURATION_Handle *cfg;
 
   /**
    * Current time to use.
    */
   struct GNUNET_TIME_Timestamp t;
 
   /**
    * Status, to be set to #GNUNET_SYSERR on failure
    */
   enum GNUNET_GenericReturnValue ret;
 };
 
 
 /**
  * Generate new denomination signing keys for the denomination type of the given @a
  * denomination_alias.
  *
  * @param cls a `struct LoadContext`, with 'ret' to be set to #GNUNET_SYSERR on failure
  * @param denomination_alias name of the denomination's section in the configuration
  */
 static void
 load_denominations (void *cls,
                     const char *denomination_alias)
 {
   struct LoadContext *ctx = cls;
   struct Denomination *denom;
   char *cipher;
 
   if ( (0 != strncasecmp (denomination_alias,
                           "coin_",
                           strlen ("coin_"))) &&
        (0 != strncasecmp (denomination_alias,
                           "coin-",
                           strlen ("coin-"))) )
     return; /* not a denomination type definition */
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_string (ctx->cfg,
                                              denomination_alias,
                                              "CIPHER",
                                              &cipher))
   {
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                                denomination_alias,
                                "CIPHER");
     return;
   }
   if (0 != strcmp (cipher, "CS"))
   {
     GNUNET_free (cipher);
     return; /* Ignore denominations of other types than CS*/
   }
   GNUNET_free (cipher);
 
   denom = GNUNET_new (struct Denomination);
   if (GNUNET_OK !=
       parse_denomination_cfg (ctx->cfg,
                               denomination_alias,
                               denom))
   {
     ctx->ret = GNUNET_SYSERR;
     GNUNET_free (denom);
     return;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
               "Loading keys for denomination %s\n",
               denom->section);
   {
     char *dname;
 
     GNUNET_asprintf (&dname,
                      "%s/%s",
                      keydir,
                      denom->section);
     GNUNET_break (GNUNET_OK ==
                   GNUNET_DISK_directory_create (dname));
     GNUNET_DISK_directory_scan (dname,
                                 &import_key,
                                 denom);
     GNUNET_free (dname);
   }
   GNUNET_CONTAINER_DLL_insert (denom_head,
                                denom_tail,
                                denom);
 }
 
 
 /**
  * Load the various duration values from @a cfg
  *
  * @param cfg configuration to use
  * @return #GNUNET_OK on success
  */
 static enum GNUNET_GenericReturnValue
 load_durations (const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   char *secname;
 
   GNUNET_asprintf (&secname,
                    "%s-secmod-cs",
                    globals->section);
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_time (cfg,
                                            secname,
                                            "OVERLAP_DURATION",
                                            &overlap_duration))
   {
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                                secname,
                                "OVERLAP_DURATION");
     GNUNET_free (secname);
     return GNUNET_SYSERR;
   }
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_time (cfg,
                                            secname,
                                            "LOOKAHEAD_SIGN",
                                            &lookahead_sign))
   {
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                                secname,
                                "LOOKAHEAD_SIGN");
     GNUNET_free (secname);
     return GNUNET_SYSERR;
   }
   GNUNET_free (secname);
   return GNUNET_OK;
 }
 
 
 /**
  * Function run on shutdown. Stops the various jobs (nicely).
  *
  * @param cls a `struct TALER_SECMOD_Options`
  */
 static void
 do_shutdown (void *cls)
 {
   (void) cls;
   TES_listen_stop ();
   if (NULL != keygen_task)
   {
     GNUNET_SCHEDULER_cancel (keygen_task);
     keygen_task = NULL;
   }
   stop_workers ();
   sem_done (&worker_sem);
 }
 
 
 void
 TALER_SECMOD_cs_run (void *cls,
                      char *const *args,
                      const char *cfgfile,
                      const struct GNUNET_CONFIGURATION_Handle *cfg)
 {
   static struct TES_Callbacks cb = {
     .dispatch = &cs_work_dispatch,
     .updater = &cs_update_client_keys,
     .init = &cs_client_init
   };
   struct TALER_SECMOD_Options *opt = cls;
   char *secname;
 
   (void) args;
   (void) cfgfile;
   globals = opt;
   if (GNUNET_TIME_timestamp_cmp (opt->global_now,
                                  !=,
                                  opt->global_now_tmp))
   {
     /* The user gave "--now", use it! */
     opt->global_now = opt->global_now_tmp;
   }
   else
   {
     /* get current time again, we may be timetraveling! */
     opt->global_now = GNUNET_TIME_timestamp_get ();
   }
   GNUNET_asprintf (&secname,
                    "%s-secmod-cs",
                    opt->section);
   if (GNUNET_OK !=
       GNUNET_CONFIGURATION_get_value_filename (cfg,
                                                secname,
                                                "KEY_DIR",
                                                &keydir))
   {
     GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
                                secname,
                                "KEY_DIR");
     GNUNET_free (secname);
     opt->global_ret = EXIT_NOTCONFIGURED;
     return;
   }
   if (GNUNET_OK !=
       load_durations (cfg))
   {
     opt->global_ret = EXIT_NOTCONFIGURED;
     GNUNET_free (secname);
     return;
   }
   opt->global_ret = TES_listen_start (cfg,
                                       secname,
                                       &cb);
   GNUNET_free (secname);
   if (0 != opt->global_ret)
     return;
   sem_init (&worker_sem,
             0);
   GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
                                  opt);
   if (0 == opt->max_workers)
   {
     long lret;
 
     lret = sysconf (_SC_NPROCESSORS_CONF);
     if (lret <= 0)
       lret = 1;
     opt->max_workers = (unsigned int) lret;
   }
   for (unsigned int i = 0; i<opt->max_workers; i++)
     if (GNUNET_OK !=
         start_worker ())
     {
       GNUNET_SCHEDULER_shutdown ();
       return;
     }
   /* Load denominations */
   keys = GNUNET_CONTAINER_multihashmap_create (65536,
                                                true);
   {
     struct LoadContext lc = {
       .cfg = cfg,
       .ret = GNUNET_OK,
       .t = opt->global_now
     };
     bool wake = true;
 
     GNUNET_assert (0 == pthread_mutex_lock (&keys_lock));
     GNUNET_CONFIGURATION_iterate_sections (cfg,
                                            &load_denominations,
                                            &lc);
     GNUNET_assert (0 == pthread_mutex_unlock (&keys_lock));
     if (GNUNET_OK != lc.ret)
     {
       opt->global_ret = EXIT_FAILURE;
       GNUNET_SCHEDULER_shutdown ();
       return;
     }
     create_missing_keys (opt,
                          &wake);
   }
   if (NULL == denom_head)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                 "No CS denominations configured. Make sure section names start with `%s' if you are using CS!\n",
                 opt->section);
     TES_wake_clients ();
     return;
   }
   /* start job to keep keys up-to-date; MUST be run before the #listen_task,
      hence with priority. */
   keygen_task = GNUNET_SCHEDULER_add_with_priority (
     GNUNET_SCHEDULER_PRIORITY_URGENT,
     &update_denominations,
     opt);
 }