paivana

HTTP paywall reverse proxy
Log | Files | Refs | Submodules | README | LICENSE

paivana-httpd_templates.c (53951B)


      1 /*
      2      This file is part of GNUnet.
      3      Copyright (C) 2026 Taler Systems SA
      4 
      5      Paivana is free software; you can redistribute it and/or
      6      modify it under the terms of the GNU Affero General Public License
      7      as published by the Free Software Foundation; either version
      8      3, or (at your option) any later version.
      9 
     10      Paivana is distributed in the hope that it will be useful,
     11      but WITHOUT ANY WARRANTY; without even the implied warranty
     12      of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See
     13      the GNU Affero General Public License for more details.
     14 
     15      You should have received a copy of the GNU Affero General Public
     16      License along with Paivana; see the file COPYING.  If not,
     17      write to the Free Software Foundation, Inc., 51 Franklin
     18      Street, Fifth Floor, Boston, MA 02110-1301, USA.
     19 */
     20 
     21 /**
     22  * @author Christian Grothoff
     23  * @file paivana-httpd_templates.c
     24  * @brief template functions
     25  */
     26 #include "platform.h"
     27 #include <curl/curl.h>
     28 #include <gnunet/gnunet_util_lib.h>
     29 #include <gnunet/gnunet_uri_lib.h>
     30 #include <gnunet/gnunet_curl_lib.h>
     31 #include "paivana-httpd.h"
     32 #include "paivana-httpd_daemon.h"
     33 #include "paivana-httpd_helper.h"
     34 #include "paivana-httpd_templates.h"
     35 #include <taler/taler_mhd_lib.h>
     36 #include <taler/taler_templating_lib.h>
     37 #include "paivana_pd.h"
     38 #include <regex.h>
     39 
     40 
     41 struct Template;
     42 #define TALER_MERCHANT_GET_PRIVATE_TEMPLATE_RESULT_CLOSURE struct Template
     43 #include <taler/merchant/get-private-templates-TEMPLATE_ID.h>
     44 #include <taler/merchant/get-private-templates.h>
     45 
     46 
     47 /**
     48  * Maximum number of rendered paywall responses cached process-wide.
     49  *
     50  * A cache key is one merchant template, one installed language and one of two
     51  * compression outcomes.  256 entries cover the common case of 128 merchant
     52  * templates in one language and both encodings, while replacing the old
     53  * per-template cap whose aggregate could grow to 16,384 responses.  The exact
     54  * byte cost depends on rendered choices, so the entry cap bounds it to 256
     55  * times the largest rendered response.  On reaching it we evict the global
     56  * least-recently-used entry.
     57  */
     58 #define MAX_RESPONSE_CACHE_ENTRIES 256
     59 
     60 /**
     61  * Maximum entries accepted from `GET /private/templates'.  1024 bounds the
     62  * temporary ID list received from a shared merchant that may contain many
     63  * non-Paivana templates; only #MAX_PAIVANA_TEMPLATES survive startup.
     64  */
     65 #define MAX_DISCOVERED_TEMPLATES 1024
     66 
     67 /**
     68  * Maximum active Paivana templates.  Every request may test their regular
     69  * expressions and every template retains contract choices, so 128 bounds
     70  * both per-request CPU and long-lived memory.
     71  */
     72 #define MAX_PAIVANA_TEMPLATES 128
     73 
     74 /**
     75  * Maximum simultaneous merchant template-detail fetches during startup.
     76  * Eight keeps startup parallel without consuming the 32-descriptor payment
     77  * reserve or producing a large burst against the merchant backend.
     78  */
     79 #define MAX_TEMPLATE_FETCHES 8
     80 
     81 /**
     82  * Largest compact JSON representation accepted for one template contract.
     83  * One MiB is ample for prices and localized summaries while bounding the
     84  * contract tree retained for rendering and rejecting accidental bulk data.
     85  */
     86 #define MAX_TEMPLATE_CONTRACT_SIZE (1024 * 1024)
     87 
     88 /**
     89  * Maximum installed paywall languages recognized for cache normalization.
     90  * Thirty-two is far above a practical site translation set and bounds both
     91  * startup metadata and the language-selection loop on every cache lookup.
     92  */
     93 #define MAX_PAYWALL_LANGUAGES 32
     94 
     95 /**
     96  * How long we give the merchant backend to answer our template
     97  * queries before abandoning the startup sequence.
     98  *
     99  * Nothing is served until they are in: #PAIVANA_HTTPD_serve_requests()
    100  * — which binds the listen sockets — is only reached from the last of
    101  * these callbacks, and neither `TALER_MERCHANT_curl_easy_get_()' nor
    102  * anything above it arms CURLOPT_TIMEOUT.  A backend that accepts the
    103  * TCP connection and then never answers would otherwise leave paivana
    104  * neither serving nor exiting, with no log line after the startup
    105  * banner; under `SERVE = systemd' the listening socket already
    106  * exists, so clients connect successfully and then hang forever with
    107  * nothing accepting them.  Generous enough for the round-trips a load
    108  * takes (GET /private/templates, then one GET per template, issued in
    109  * batches of at most eight).
    110  */
    111 #define TEMPLATE_LOAD_TIMEOUT \
    112         GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 2)
    113 
    114 
    115 /**
    116  * Entry in the cache of responses for a given template.
    117  */
    118 struct ResponseCacheEntry
    119 {
    120 
    121   /**
    122    * Kept in a DLL.
    123    */
    124   struct ResponseCacheEntry *next;
    125 
    126   /**
    127    * Kept in a DLL.
    128    */
    129   struct ResponseCacheEntry *prev;
    130 
    131   /** Merchant template this rendered response belongs to. */
    132   struct Template *template;
    133 
    134   /**
    135    * Language of the response.
    136    */
    137   char *lang;
    138 
    139   /**
    140    * True if @e paywall carries a deflate-compressed body.
    141    */
    142   bool deflate;
    143 
    144   /**
    145    * Paywall response for these request parameters.
    146    */
    147   struct MHD_Response *paywall;
    148 
    149   /**
    150    * HTTP status to return with @e paywall.
    151    */
    152   unsigned int http_status;
    153 
    154 };
    155 
    156 
    157 /**
    158  * Information about a template in the merchant backend.
    159  */
    160 struct Template
    161 {
    162 
    163   /**
    164    * Kept in a DLL.
    165    */
    166   struct Template *next;
    167 
    168   /**
    169    * Kept in a DLL.
    170    */
    171   struct Template *prev;
    172 
    173   /**
    174    * ID of the template.
    175    */
    176   char *template_id;
    177 
    178   /**
    179    * Summary of the template, NULL if not given.
    180    */
    181   char *summary;
    182 
    183   /**
    184    * Maximum pickup delay for the pages.
    185    */
    186   struct GNUNET_TIME_Relative max_pickup_delay;
    187 
    188   /**
    189    * Ways how to pay for the template.
    190    */
    191   json_t *choices;
    192 
    193   /**
    194    * Regular expression of websites the template is for.
    195    */
    196   char *regex;
    197 
    198   /**
    199    * Pre-compiled regular expression @e regex.
    200    */
    201   regex_t ex;
    202 
    203   /**
    204    * Handle used to request more information about the template.
    205    */
    206   struct TALER_MERCHANT_GetPrivateTemplateHandle *gt;
    207 
    208   /**
    209    * Startup fetch state.  Detail requests are started in bounded batches
    210    * rather than all at once.
    211    */
    212   enum
    213   {
    214     TLS_WAITING,
    215     TLS_ACTIVE,
    216     TLS_DONE
    217   } load_state;
    218 
    219 };
    220 
    221 
    222 /**
    223  * Kept in a DLL.
    224  */
    225 static struct Template *t_head;
    226 
    227 /**
    228  * Kept in a DLL.
    229  */
    230 static struct Template *t_tail;
    231 
    232 /** Head of the process-wide rendered-response LRU. */
    233 static struct ResponseCacheEntry *rce_head;
    234 
    235 /** Tail of the process-wide rendered-response LRU. */
    236 static struct ResponseCacheEntry *rce_tail;
    237 
    238 /** Number of entries in the process-wide rendered-response LRU. */
    239 static unsigned int rce_length;
    240 
    241 /** Installed language tags for the `paywall' Mustache template. */
    242 static char *paywall_languages[MAX_PAYWALL_LANGUAGES];
    243 
    244 /** Number of entries in #paywall_languages. */
    245 static unsigned int paywall_languages_length;
    246 
    247 /** Template IDs waiting for a detail request to start. */
    248 static unsigned int pending_template_fetches;
    249 
    250 /** Template detail requests currently in flight. */
    251 static unsigned int active_template_fetches;
    252 
    253 /** Paivana templates retained after their contracts were inspected. */
    254 static unsigned int loaded_paivana_templates;
    255 
    256 /** Next waiting template, making bounded fetch dispatch O(T). */
    257 static struct Template *next_template_fetch;
    258 
    259 /**
    260  * Handle to get all the templates.
    261  */
    262 static struct TALER_MERCHANT_GetPrivateTemplatesHandle *gpt;
    263 
    264 /**
    265  * Watchdog for #TEMPLATE_LOAD_TIMEOUT, NULL once the templates are in
    266  * (or once we have given up on them).
    267  */
    268 static struct GNUNET_SCHEDULER_Task *load_timeout_task;
    269 
    270 
    271 /**
    272  * Task run when the merchant backend did not answer our template
    273  * queries within #TEMPLATE_LOAD_TIMEOUT.
    274  *
    275  * Treated exactly like any other failure to load the templates (an
    276  * unauthorized or unexpected status from the backend): the daemon
    277  * exits with a diagnosis instead of stalling.  Whether that policy is
    278  * the right one is a separate question — this task only makes sure the
    279  * stall is not a third, silent outcome.
    280  *
    281  * @param cls NULL
    282  */
    283 static void
    284 load_timeout (void *cls)
    285 {
    286   (void) cls;
    287   load_timeout_task = NULL;
    288   GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
    289               "Merchant backend at `%s' did not answer our template queries"
    290               " within %s; giving up instead of never starting to serve\n",
    291               PH_merchant_internal_url,
    292               GNUNET_STRINGS_relative_time_to_string (TEMPLATE_LOAD_TIMEOUT,
    293                                                       true));
    294   PH_global_ret = EXIT_FAILURE;
    295   GNUNET_SCHEDULER_shutdown ();
    296 }
    297 
    298 
    299 /**
    300  * The templates are in: stop the watchdog and open the listen sockets.
    301  *
    302  * Refuses to start if there is not a single one.  Only reachable with
    303  * the paywall enabled -- `-n' returns before
    304  * #PAIVANA_HTTPD_load_templates() is ever called -- and a paywall with
    305  * nothing to sell is not a paywall: PAIVANA_HTTPD_search_templates()
    306  * would answer "no template matched" for every URL, which create_response()
    307  * reads as "no paywall applies" and forwards.  The whole site would be
    308  * free, silently, which is exactly the outcome an operator running a
    309  * paywall did not ask for.  An operator who does want a plain reverse
    310  * proxy says so with `-n'.
    311  */
    312 static void
    313 templates_ready (void)
    314 {
    315   if (NULL == t_head)
    316   {
    317     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
    318                 "The merchant backend at `%s' offers no paivana template;"
    319                 " refusing to start, as every request would then be"
    320                 " forwarded for free.  Pass -n if serving the site without"
    321                 " a paywall is what you want.\n",
    322                 PH_merchant_internal_url);
    323     PH_global_ret = EXIT_NOTCONFIGURED;
    324     GNUNET_SCHEDULER_shutdown ();
    325     return;
    326   }
    327   if (NULL != load_timeout_task)
    328   {
    329     GNUNET_SCHEDULER_cancel (load_timeout_task);
    330     load_timeout_task = NULL;
    331   }
    332   PAIVANA_HTTPD_serve_requests ();
    333 }
    334 
    335 
    336 /**
    337  * Check if two strings are equal, including both being NULL
    338  *
    339  * @param s1 a string, possibly NULL
    340  * @param s2 a string. possibly NULL
    341  * @return true if both are equal
    342  */
    343 static bool
    344 eq (const char *s1,
    345     const char *s2)
    346 {
    347   if (s1 == s2)
    348     return true;
    349   if (NULL == s1)
    350     return false;
    351   if (NULL == s2)
    352     return false;
    353   return (0 == strcmp (s1,
    354                        s2));
    355 }
    356 
    357 
    358 /**
    359  * Create a taler://pay-template/ URI for the given @a con and @a template_id
    360  * and @a instance_id.
    361  *
    362  * @param merchant_base_url URL to take host and path from;
    363  *        we cannot take it from the MHD connection as a browser
    364  *        may have changed 'http' to 'https' and we MUST be consistent
    365  *        with what the merchant's frontend used initially
    366  * @param template_id the template id
    367  * @return corresponding taler://pay-template/ URI, or NULL on missing "host"
    368  */
    369 static char *
    370 make_taler_pay_template_uri (const char *merchant_base_url,
    371                              const char *template_id)
    372 {
    373   struct GNUNET_Buffer buf = { 0 };
    374   char *url;
    375   struct GNUNET_Uri uri;
    376 
    377   url = GNUNET_strdup (merchant_base_url);
    378   if (-1 == GNUNET_uri_parse (&uri,
    379                               url))
    380   {
    381     GNUNET_break (0);
    382     GNUNET_free (url);
    383     return NULL;
    384   }
    385   if ( (NULL == uri.scheme) ||
    386        (NULL == uri.host) )
    387   {
    388     GNUNET_break (0);
    389     GNUNET_free (url);
    390     return NULL;
    391   }
    392   GNUNET_assert (NULL != template_id);
    393   GNUNET_buffer_write_str (&buf,
    394                            "taler");
    395   if (0 == strcasecmp ("http",
    396                        uri.scheme))
    397     GNUNET_buffer_write_str (&buf,
    398                              "+http");
    399   GNUNET_buffer_write_str (&buf,
    400                            "://pay-template/");
    401   GNUNET_buffer_write_str (&buf,
    402                            uri.host);
    403   if (0 != uri.port)
    404     GNUNET_buffer_write_fstr (&buf,
    405                               ":%u",
    406                               (unsigned int) uri.port);
    407   if (NULL != uri.path)
    408     GNUNET_buffer_write_path (&buf,
    409                               uri.path);
    410   GNUNET_buffer_write_path (&buf,
    411                             template_id);
    412   GNUNET_free (url);
    413   return GNUNET_buffer_reap_str (&buf);
    414 }
    415 
    416 
    417 /**
    418  * Render @a s as a complete, double-quoted JavaScript string literal.
    419  *
    420  * The paywall page carries its context in `const' declarations inside a
    421  * <script> element, and mustache's default escaping is the wrong
    422  * escaping there twice over: it escapes exactly '<', '>', '&' and '"'
    423  * (mustach-wrap.c), which leaves the apostrophe free to close a
    424  * single-quoted literal and let arbitrary JavaScript follow, and the
    425  * entity references it does produce are never decoded, because the HTML
    426  * parser does not decode them inside a raw-text element -- an '&' in the
    427  * merchant base URL reached the script as the literal text "&amp;".
    428  *
    429  * So the value is escaped for the context it actually lands in, and
    430  * interpolated with the unescaped {{{ }}} since it arrives complete with
    431  * its quotes.  '<', '>' and '&' are still escaped, as \\uXXXX rather
    432  * than as entities: without that a value containing "</script>" would
    433  * end the element regardless of how well the string literal itself is
    434  * quoted.
    435  *
    436  * @param s string to render, must be valid UTF-8
    437  * @return JavaScript literal including the surrounding quotes,
    438  *         to be freed by the caller
    439  */
    440 static char *
    441 js_string_literal (const char *s)
    442 {
    443   struct GNUNET_Buffer buf = { 0 };
    444 
    445   GNUNET_buffer_write_str (&buf,
    446                            "\"");
    447   for (const unsigned char *p = (const unsigned char *) s;
    448        '\0' != *p;
    449        p++)
    450   {
    451     switch (*p)
    452     {
    453     case '"':
    454       GNUNET_buffer_write_str (&buf,
    455                                "\\\"");
    456       break;
    457     case '\\':
    458       GNUNET_buffer_write_str (&buf,
    459                                "\\\\");
    460       break;
    461     default:
    462       if ( (*p < 0x20) ||
    463            (0x7F == *p) ||
    464            ('<' == *p) ||
    465            ('>' == *p) ||
    466            ('&' == *p) )
    467         GNUNET_buffer_write_fstr (&buf,
    468                                   "\\u%04x",
    469                                   (unsigned int) *p);
    470       else
    471         GNUNET_buffer_write_fstr (&buf,
    472                                   "%c",
    473                                   (char) *p);
    474       break;
    475     }
    476   }
    477   GNUNET_buffer_write_str (&buf,
    478                            "\"");
    479   return GNUNET_buffer_reap_str (&buf);
    480 }
    481 
    482 
    483 /**
    484  * Record the language of an installed `paywall.$LANG.must' template.
    485  *
    486  * The scan intentionally uses the same directory and filename grammar as
    487  * TALER_TEMPLATING_init().  Directory traversal order is stable across the
    488  * two consecutive scans, preserving the templating library's first-template
    489  * fallback when no language matches.
    490  *
    491  * @param cls unused
    492  * @param filename file found in the template directory
    493  * @return #GNUNET_OK to continue, #GNUNET_SYSERR on the language cap
    494  */
    495 static enum GNUNET_GenericReturnValue
    496 collect_paywall_language (void *cls,
    497                           const char *filename)
    498 {
    499   const char *base;
    500   const char *lang;
    501   const char *end;
    502 
    503   (void) cls;
    504   base = strrchr (filename,
    505                   '/');
    506   base = (NULL == base) ? filename : base + 1;
    507   if (0 != strncmp (base,
    508                     "paywall.",
    509                     strlen ("paywall.")))
    510     return GNUNET_OK;
    511   lang = base + strlen ("paywall.");
    512   end = strchr (lang,
    513                 '.');
    514   if ( (lang == end) ||
    515        (NULL == end) ||
    516        (0 != strcmp (end,
    517                      ".must")) )
    518     return GNUNET_OK;
    519   if (paywall_languages_length >= MAX_PAYWALL_LANGUAGES)
    520   {
    521     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
    522                 "More than %u paywall languages are installed; refusing"
    523                 " startup because language selection and cache variation"
    524                 " would exceed the documented bound\n",
    525                 (unsigned int) MAX_PAYWALL_LANGUAGES);
    526     return GNUNET_SYSERR;
    527   }
    528   paywall_languages[paywall_languages_length++]
    529     = GNUNET_strndup (lang,
    530                       end - lang);
    531   return GNUNET_OK;
    532 }
    533 
    534 
    535 bool
    536 PAIVANA_HTTPD_init_template_languages (void)
    537 {
    538   char *dir;
    539   char *tdir;
    540   int ret;
    541 
    542   GNUNET_assert (0 == paywall_languages_length);
    543   dir = GNUNET_OS_installation_get_path (PAIVANA_project_data (),
    544                                          GNUNET_OS_IPK_DATADIR);
    545   GNUNET_asprintf (&tdir,
    546                    "%stemplates",
    547                    dir);
    548   GNUNET_free (dir);
    549   ret = GNUNET_DISK_directory_scan (tdir,
    550                                     &collect_paywall_language,
    551                                     NULL);
    552   if ( (0 > ret) ||
    553        (0 == paywall_languages_length) )
    554   {
    555     if (0 == paywall_languages_length)
    556       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
    557                   "No paywall.$LANG.must template is installed in `%s'\n",
    558                   tdir);
    559     GNUNET_free (tdir);
    560     return false;
    561   }
    562   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
    563               "%u paywall language%s installed in `%s'\n",
    564               paywall_languages_length,
    565               (1 == paywall_languages_length) ? "" : "s",
    566               tdir);
    567   GNUNET_free (tdir);
    568   return true;
    569 }
    570 
    571 
    572 /**
    573  * The `Content-Security-Policy' for the paywall page, built once from
    574  * #PH_merchant_base_url.  NULL until first needed.
    575  */
    576 static char *paywall_csp;
    577 
    578 
    579 /**
    580  * Return the `Content-Security-Policy' for the paywall page.
    581  *
    582  * The page is entirely self-contained -- every stylesheet, script and
    583  * image is inline -- so everything but the merchant backend it polls
    584  * can be denied outright.  That is the part worth having: with
    585  * `default-src' at 'none' and `connect-src' naming exactly two origins,
    586  * script that does run cannot reach an attacker's host to report what
    587  * it found, and `frame-ancestors' keeps the taler:// link from being
    588  * framed and clicked by proxy.
    589  *
    590  * `script-src' still has to permit inline script: the page carries three
    591  * inline <script> elements and one `onclick' attribute, and neither
    592  * hashes nor a nonce survive our response cache, which serves one
    593  * rendered body to every client for five minutes.  Removing
    594  * 'unsafe-inline' means moving that handler into paywall.js and hashing
    595  * each block after rendering; worth doing, but it is not what makes the
    596  * injection this policy backs up impossible -- js_string_literal() is.
    597  *
    598  * @return the policy, owned by this module, or NULL if the merchant
    599  *         base URL cannot be parsed
    600  */
    601 static const char *
    602 get_paywall_csp (void)
    603 {
    604   struct GNUNET_Buffer buf = { 0 };
    605   struct GNUNET_Uri uri;
    606   char *url;
    607 
    608   if (NULL != paywall_csp)
    609     return paywall_csp;
    610   url = GNUNET_strdup (PH_merchant_base_url);
    611   if ( (-1 == GNUNET_uri_parse (&uri,
    612                                 url)) ||
    613        (NULL == uri.scheme) ||
    614        (NULL == uri.host) )
    615   {
    616     /* Cannot name the backend, and a policy that omits it would break
    617        the polling the page exists to do.  Serve without one rather than
    618        with a broken one; the URL is checked at startup, so this is the
    619        unreachable arm. */
    620     GNUNET_break (0);
    621     GNUNET_free (url);
    622     return NULL;
    623   }
    624   GNUNET_buffer_write_str (&buf,
    625                            "default-src 'none'; "
    626                            "script-src 'unsafe-inline'; "
    627                            "style-src 'unsafe-inline'; "
    628                            /* the QR code is drawn to a canvas and
    629                               handed to an <img> as a data: URL */
    630                            "img-src data:; "
    631                            "connect-src 'self' ");
    632   GNUNET_buffer_write_str (&buf,
    633                            uri.scheme);
    634   GNUNET_buffer_write_str (&buf,
    635                            "://");
    636   GNUNET_buffer_write_str (&buf,
    637                            uri.host);
    638   if (0 != uri.port)
    639     GNUNET_buffer_write_fstr (&buf,
    640                               ":%u",
    641                               (unsigned int) uri.port);
    642   GNUNET_buffer_write_str (&buf,
    643                            "; frame-ancestors 'none'"
    644                            "; base-uri 'none'"
    645                            "; form-action 'none'");
    646   GNUNET_free (url);
    647   paywall_csp = GNUNET_buffer_reap_str (&buf);
    648   return paywall_csp;
    649 }
    650 
    651 
    652 /**
    653  * Return the language component of the render cache key for @a conn.
    654  *
    655  * Reproduce TALER_TEMPLATING_build()'s language choice from the installed
    656  * language list.  The cache is therefore keyed by one of at most 32 actual
    657  * variants instead of by an unbounded client-controlled header string.
    658  *
    659  * @param conn connection to derive the key component for
    660  * @return the key component, or NULL if `Accept-Language' does not
    661  *         affect the rendered body
    662  */
    663 static const char *
    664 cache_key_language (struct MHD_Connection *conn)
    665 {
    666   const char *pattern;
    667   const char *best = NULL;
    668   double best_q = 0.0;
    669 
    670   GNUNET_assert (0 != paywall_languages_length);
    671   pattern = MHD_lookup_connection_value (conn,
    672                                          MHD_HEADER_KIND,
    673                                          MHD_HTTP_HEADER_ACCEPT_LANGUAGE);
    674   if (NULL == pattern)
    675     pattern = "en";
    676   for (unsigned int i = 0; i < paywall_languages_length; i++)
    677   {
    678     double q = TALER_pattern_matches (pattern,
    679                                       paywall_languages[i]);
    680 
    681     if (q <= best_q)
    682       continue;
    683     best_q = q;
    684     best = paywall_languages[i];
    685   }
    686   /* This is the same first-loaded fallback as lookup_template() in the
    687      templating library. */
    688   return (NULL != best) ? best : paywall_languages[0];
    689 }
    690 
    691 
    692 /**
    693  * Try to initialize the paywall response.
    694  *
    695  * @param conn connection to create the response for
    696  * @param t template to create the response for
    697  * @return MHD status code to return
    698  */
    699 static enum MHD_Result
    700 load_paywall (struct MHD_Connection *conn,
    701               struct Template *t)
    702 {
    703   struct MHD_Response *reply;
    704   const char *lang;
    705   bool deflate;
    706   unsigned int http_status = MHD_HTTP_PAYMENT_REQUIRED;
    707 
    708   lang = cache_key_language (conn);
    709   /* `Accept-Encoding' reaches the body through exactly this predicate
    710      (templating_api.c), so keying on its result rather than on the
    711      header text is not an approximation: it is the decision itself,
    712      and it has two outcomes instead of unboundedly many. */
    713   deflate = (TALER_MHD_CT_DEFLATE ==
    714              TALER_MHD_can_compress (conn,
    715                                      TALER_MHD_CT_DEFLATE));
    716   for (struct ResponseCacheEntry *pos = rce_head;
    717        NULL != pos;
    718        pos = pos->next)
    719   {
    720     if ( (t == pos->template) &&
    721          (eq (lang,
    722               pos->lang)) &&
    723          (deflate == pos->deflate) )
    724     {
    725       if (rce_head != pos)
    726       {
    727         /* Hit, move pos to head of DLL for proper LRU eviction */
    728         GNUNET_CONTAINER_DLL_remove (rce_head,
    729                                      rce_tail,
    730                                      pos);
    731         GNUNET_CONTAINER_DLL_insert (rce_head,
    732                                      rce_tail,
    733                                      pos);
    734       }
    735       return MHD_queue_response (conn,
    736                                  pos->http_status,
    737                                  pos->paywall);
    738     }
    739   }
    740 
    741   {
    742     enum GNUNET_GenericReturnValue ret;
    743     json_t *data;
    744     char *tid_js = js_string_literal (t->template_id);
    745     char *mb_js = js_string_literal (PH_merchant_base_url);
    746 
    747     data = GNUNET_JSON_PACK (
    748       GNUNET_JSON_pack_string (
    749         "template_id",
    750         t->template_id),
    751       /* The `_js' variants are complete JavaScript string literals,
    752          quotes included, for the <script> block; the plain ones are for
    753          the HTML body, where mustache's own escaping is correct. */
    754       GNUNET_JSON_pack_string (
    755         "template_id_js",
    756         tid_js),
    757       GNUNET_JSON_pack_string (
    758         "merchant_backend_js",
    759         mb_js),
    760       GNUNET_JSON_pack_allow_null (
    761         GNUNET_JSON_pack_string (
    762           "summary",
    763           t->summary)),
    764       GNUNET_JSON_pack_allow_null (
    765         GNUNET_JSON_pack_array_incref (
    766           "choices",
    767           t->choices)),
    768       GNUNET_JSON_pack_bool (
    769         "has_choices",
    770         1 < json_array_size (t->choices)),
    771       GNUNET_JSON_pack_allow_null (
    772         GNUNET_JSON_pack_object_incref (
    773           "default_choice",
    774           json_array_get (t->choices, 0))),
    775       GNUNET_JSON_pack_uint64 (
    776         "max_pickup_delay",
    777         /* Note: 'forever' will result in a very large number
    778            here, that is intentional, it is equivalent and avoids
    779            a special case on the client-side. */
    780         t->max_pickup_delay.rel_value_us / 1000LLU / 1000LLU),
    781       GNUNET_JSON_pack_string (
    782         "merchant_backend",
    783         PH_merchant_base_url));
    784     GNUNET_free (tid_js);
    785     GNUNET_free (mb_js);
    786     ret = TALER_TEMPLATING_build (
    787       conn,
    788       &http_status,
    789       "paywall",
    790       NULL /* no instance */,
    791       NULL /* no Taler URI (needs dynamic paivana_id!) */,
    792       data,
    793       &reply);
    794     json_decref (data);
    795     if (GNUNET_NO == ret)
    796     {
    797       enum MHD_Result mret;
    798 
    799       /* taler_templating_lib.h: #GNUNET_NO means an *error reply* was
    800          built — typically because the paywall template is not
    801          installed where TALER_TEMPLATING_init() looked.  It is a live
    802          MHD_Response and it is ours: returning #MHD_YES without
    803          queuing it leaked it once per unauthenticated request and left
    804          the client waiting for a status line that never came. */
    805       GNUNET_break (0);
    806       mret = MHD_queue_response (conn,
    807                                  http_status,
    808                                  reply);
    809       MHD_destroy_response (reply);
    810       return mret;
    811     }
    812     if (GNUNET_OK != ret)
    813     {
    814       /* #GNUNET_SYSERR: no reply was built, so there is nothing to
    815          queue and nothing to free; MHD must close the connection. */
    816       GNUNET_break (0);
    817       return MHD_NO;
    818     }
    819   }
    820 
    821 
    822   GNUNET_break (MHD_YES ==
    823                 MHD_add_response_header (reply,
    824                                          MHD_HTTP_HEADER_CONTENT_TYPE,
    825                                          "text/html"));
    826   /* The paywall body depends on the negotiated language and on
    827      whether we deflated it for the client; tell intermediaries to
    828      key their cache entries on both. */
    829   GNUNET_break (MHD_YES ==
    830                 MHD_add_response_header (reply,
    831                                          MHD_HTTP_HEADER_VARY,
    832                                          MHD_HTTP_HEADER_ACCEPT_LANGUAGE ", "
    833                                          MHD_HTTP_HEADER_ACCEPT_ENCODING ", "
    834                                          "Cookie"));
    835   GNUNET_break (MHD_YES ==
    836                 MHD_add_response_header (reply,
    837                                          MHD_HTTP_HEADER_CACHE_CONTROL,
    838                                          "public, max-age=300"));
    839   {
    840     const char *csp = get_paywall_csp ();
    841 
    842     if (NULL != csp)
    843       GNUNET_break (MHD_YES ==
    844                     MHD_add_response_header (reply,
    845                                              MHD_HTTP_HEADER_CONTENT_SECURITY_POLICY,
    846                                              csp));
    847   }
    848   /* frame-ancestors covers this for anything current; X-Frame-Options
    849      is for the user agents that do not implement it. */
    850   GNUNET_break (MHD_YES ==
    851                 MHD_add_response_header (reply,
    852                                          MHD_HTTP_HEADER_X_FRAME_OPTIONS,
    853                                          "DENY"));
    854   GNUNET_break (MHD_YES ==
    855                 MHD_add_response_header (reply,
    856                                          MHD_HTTP_HEADER_X_CONTENT_TYPE_OPTIONS,
    857                                          "nosniff"));
    858   {
    859     char *uri;
    860 
    861     uri = make_taler_pay_template_uri (PH_merchant_base_url,
    862                                        t->template_id);
    863     if (NULL != uri)
    864     {
    865       GNUNET_break (MHD_YES ==
    866                     MHD_add_response_header (reply,
    867                                              "Paivana",
    868                                              uri));
    869       GNUNET_free (uri);
    870     }
    871   }
    872 
    873   {
    874     struct ResponseCacheEntry *rce;
    875 
    876     /* '>=', not '>': the insert below is what takes us to the cap, so
    877        testing '>' left the steady state one entry above it. */
    878     while (rce_length >= MAX_RESPONSE_CACHE_ENTRIES)
    879     {
    880       /* Evict the least recently used entry; the hit path above
    881          promotes to the head, so the tail is the coldest. */
    882       struct ResponseCacheEntry *old = rce_tail;
    883 
    884       GNUNET_CONTAINER_DLL_remove (rce_head,
    885                                    rce_tail,
    886                                    old);
    887       GNUNET_assert (rce_length > 0);
    888       rce_length--;
    889       MHD_destroy_response (old->paywall);
    890       GNUNET_free (old->lang);
    891       GNUNET_free (old);
    892     }
    893     rce = GNUNET_new (struct ResponseCacheEntry);
    894     rce->template = t;
    895     if (NULL != lang)
    896       rce->lang = GNUNET_strdup (lang);
    897     rce->deflate = deflate;
    898     rce->paywall = reply;
    899     rce->http_status = http_status;
    900     rce_length++;
    901     GNUNET_CONTAINER_DLL_insert (rce_head,
    902                                  rce_tail,
    903                                  rce);
    904     return MHD_queue_response (conn,
    905                                rce->http_status,
    906                                reply);
    907   }
    908 }
    909 
    910 
    911 /**
    912  * Parse template contract to (mostly) determine the
    913  * regex specifying which websites the template applies to.
    914  *
    915  * @param[in,out] t template to update
    916  * @param contract contract to parse
    917  * @return true on success, false on failure
    918  */
    919 static bool
    920 parse_template (struct Template *t,
    921                 const json_t *contract)
    922 {
    923   /* An absent website_regex means "every URL".  So does an empty one:
    924      it is what a merchant UI stores for a template that was never
    925      given a restriction, and anchoring it would yield "^()$", which
    926      matches only the empty string and therefore no URL at all -- a
    927      template that silently paywalls nothing. */
    928   const char *regex = NULL;
    929   const char *summary = NULL;
    930   const json_t *choices = NULL;
    931   struct GNUNET_JSON_Specification spec[] = {
    932     GNUNET_JSON_spec_mark_optional (
    933       GNUNET_JSON_spec_string ("website_regex",
    934                                &regex),
    935       NULL),
    936     GNUNET_JSON_spec_mark_optional (
    937       GNUNET_JSON_spec_string ("summary",
    938                                &summary),
    939       NULL),
    940     GNUNET_JSON_spec_array_const ("choices",
    941                                   &choices),
    942     GNUNET_JSON_spec_mark_optional (
    943       GNUNET_JSON_spec_relative_time ("max_pickup_duration",
    944                                       &t->max_pickup_delay),
    945       NULL),
    946     GNUNET_JSON_spec_end ()
    947   };
    948   const char *en;
    949 
    950   if (GNUNET_OK !=
    951       GNUNET_JSON_parse ((json_t *) contract,
    952                          spec,
    953                          &en,
    954                          NULL))
    955   {
    956     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
    957                 "Invalid template %s at field %s\n",
    958                 t->template_id,
    959                 en);
    960     return false;
    961   }
    962   /* GNUNET_JSON_spec_array_const() only establishes that "choices" is
    963      an array; what is *in* it is whatever the backend sent.
    964      load_paywall() hands element 0 to
    965      GNUNET_JSON_pack_object_incref(), which aborts on anything that is
    966      not an object — and it does so from an unauthenticated GET of the
    967      paywall page, so a backend that disagrees with us about the schema
    968      would turn every visitor into a crash loop.  We cross a network
    969      trust boundary here and must not assert on what comes back. */
    970   {
    971     size_t idx;
    972     json_t *choice;
    973 
    974     json_array_foreach ((json_t *) choices, idx, choice)
    975     {
    976       if (json_is_object (choice))
    977         continue;
    978       GNUNET_break_op (0);
    979       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
    980                   "Template %s has a non-object choice at index %u\n",
    981                   t->template_id,
    982                   (unsigned int) idx);
    983       return false;
    984     }
    985   }
    986   if ( (NULL != regex) &&
    987        ('\0' != regex[0]) )
    988   {
    989     char *anchored;
    990     regex_t bare;
    991     int rc;
    992 
    993     /* Compile the merchant's expression as it stands *before* wrapping
    994        it, and refuse the template if that fails.  Splicing into a
    995        group is not a syntactic no-op: "a)|(b" becomes "^(a)|(b)$",
    996        which is "^a" OR "b$" -- each anchored on one side only, so the
    997        anchoring below stops being the guarantee the manual states.
    998        Every such breakout needs a parenthesis that only balances
    999        against the ones we add, which is exactly what a bare regcomp()
   1000        rejects.  The merchant backend compiles the bare expression on
   1001        POST /private/templates for the same reason; this closes the gap
   1002        for a template that reached the database some other way.  Note
   1003        that no expression that compiles on its own changes meaning
   1004        here. */
   1005     rc = regcomp (&bare,
   1006                   regex,
   1007                   REG_NOSUB | REG_EXTENDED);
   1008     if (0 != rc)
   1009     {
   1010       GNUNET_break_op (0);
   1011       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1012                   "Invalid regex in template %s: %s\n",
   1013                   t->template_id,
   1014                   regex);
   1015       return false;
   1016     }
   1017     regfree (&bare);
   1018     /* Anchor the merchant's expression: regexec(3) is unanchored, so
   1019        an expression like "/premium/" would otherwise put a paywall on
   1020        every URL merely *containing* it.  Wrapping in a group keeps
   1021        alternations ("a|b") from binding the anchors to only the first
   1022        and last branch.  An expression that already anchors itself is
   1023        unaffected, as ^ and $ inside still match at string
   1024        start/end. */
   1025     GNUNET_asprintf (&anchored,
   1026                      "^(%s)$",
   1027                      regex);
   1028     rc = regcomp (&t->ex,
   1029                   anchored,
   1030                   REG_NOSUB | REG_EXTENDED);
   1031     GNUNET_free (anchored);
   1032     if (0 != rc)
   1033     {
   1034       GNUNET_break_op (0);
   1035       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1036                   "Invalid regex in template %s: %s\n",
   1037                   t->template_id,
   1038                   regex);
   1039       return false;
   1040     }
   1041     t->regex = GNUNET_strdup (regex);
   1042   }
   1043   if (NULL != summary)
   1044     t->summary = GNUNET_strdup (summary);
   1045   t->choices = json_incref ((json_t *) choices);
   1046   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
   1047               "Using payment template %s for `%s'\n",
   1048               t->template_id,
   1049               (NULL != t->regex) ? t->regex : "(all URLs)");
   1050   return true;
   1051 }
   1052 
   1053 
   1054 /**
   1055  * Is @a contract a template that is ours to serve?
   1056  *
   1057  * The merchant's `template_type' discriminator; the literals are
   1058  * TALER_MERCHANT_template_type_from_string(), which we would call if
   1059  * it did not live in a library paivana does not otherwise need.  Note
   1060  * that an absent `template_type' is `fixed-order' there, so it is not
   1061  * ours either.
   1062  *
   1063  * @param contract template contract from the backend
   1064  * @return true if this is a paivana template
   1065  */
   1066 static bool
   1067 is_paivana_template (const json_t *contract)
   1068 {
   1069   const json_t *tt;
   1070 
   1071   tt = json_object_get (contract,
   1072                         "template_type");
   1073   return ( (NULL != tt) &&
   1074            (json_is_string (tt)) &&
   1075            (0 == strcmp ("paivana",
   1076                          json_string_value (tt))) );
   1077 }
   1078 
   1079 
   1080 /**
   1081  * Remove @a t from the list of templates and free it.
   1082  *
   1083  * Only for a template we decided not to use before anything was parsed
   1084  * into it: @e gt must already be NULL and nothing but the ID
   1085  * allocated.
   1086  *
   1087  * @param[in] t template to drop
   1088  */
   1089 static void
   1090 drop_template (struct Template *t)
   1091 {
   1092   GNUNET_assert (NULL == t->gt);
   1093   GNUNET_CONTAINER_DLL_remove (t_head,
   1094                                t_tail,
   1095                                t);
   1096   GNUNET_free (t->template_id);
   1097   GNUNET_free (t);
   1098 }
   1099 
   1100 
   1101 static void start_template_fetches (void);
   1102 
   1103 
   1104 /** Compare template pointers by ID for qsort(). */
   1105 static int
   1106 compare_template_ids (const void *a,
   1107                       const void *b)
   1108 {
   1109   const struct Template *const *ta = a;
   1110   const struct Template *const *tb = b;
   1111 
   1112   return strcmp ((*ta)->template_id,
   1113                  (*tb)->template_id);
   1114 }
   1115 
   1116 
   1117 /**
   1118  * Sort the retained templates once, after collection and filtering.
   1119  *
   1120  * Where expressions overlap, list order chooses the quoted price.  Sorting by
   1121  * ID makes that decision deterministic.  Collecting into an array and using
   1122  * qsort() is O(T log T), unlike the previous sorted DLL insertion's O(T^2).
   1123  */
   1124 static void
   1125 sort_templates (void)
   1126 {
   1127   struct Template **templates;
   1128   unsigned int i = 0;
   1129 
   1130   if (2 > loaded_paivana_templates)
   1131     return;
   1132   templates = GNUNET_malloc (loaded_paivana_templates
   1133                              * sizeof (*templates));
   1134   for (struct Template *t = t_head; NULL != t; t = t->next)
   1135     templates[i++] = t;
   1136   GNUNET_assert (i == loaded_paivana_templates);
   1137   qsort (templates,
   1138          loaded_paivana_templates,
   1139          sizeof (*templates),
   1140          &compare_template_ids);
   1141   t_head = NULL;
   1142   t_tail = NULL;
   1143   for (i = 0; i < loaded_paivana_templates; i++)
   1144   {
   1145     templates[i]->next = NULL;
   1146     templates[i]->prev = NULL;
   1147     GNUNET_CONTAINER_DLL_insert_tail (t_head,
   1148                                       t_tail,
   1149                                       templates[i]);
   1150   }
   1151   GNUNET_free (templates);
   1152 }
   1153 
   1154 
   1155 /** Finish template startup once no detail request is queued or active. */
   1156 static void
   1157 finish_template_loading (void)
   1158 {
   1159   if ( (0 != pending_template_fetches) ||
   1160        (0 != active_template_fetches) )
   1161     return;
   1162   sort_templates ();
   1163   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
   1164               "%u Paivana template%s loaded, starting to serve requests\n",
   1165               loaded_paivana_templates,
   1166               (1 == loaded_paivana_templates) ? "" : "s");
   1167   templates_ready ();
   1168 }
   1169 
   1170 
   1171 /**
   1172  * Callback for a GET /private/templates/$TEMPLATE_ID request.
   1173  *
   1174  * @param t template the request was about
   1175  * @param tgr response details
   1176  */
   1177 static void
   1178 setup_template (
   1179   struct Template *t,
   1180   const struct TALER_MERCHANT_GetPrivateTemplateResponse *tgr)
   1181 {
   1182   GNUNET_assert (TLS_ACTIVE == t->load_state);
   1183   GNUNET_assert (active_template_fetches > 0);
   1184   active_template_fetches--;
   1185   t->gt = NULL;
   1186   t->load_state = TLS_DONE;
   1187   switch (tgr->hr.http_status)
   1188   {
   1189   case MHD_HTTP_OK:
   1190     {
   1191       const json_t *contract = tgr->details.ok.template_contract;
   1192       size_t contract_size;
   1193 
   1194       if (! is_paivana_template (contract))
   1195       {
   1196         /* A shared merchant serves other template types too; they count only
   1197            against the broad discovery bound and are discarded here. */
   1198         GNUNET_log (GNUNET_ERROR_TYPE_INFO,
   1199                     "Ignoring template %s: not a paivana template\n",
   1200                     t->template_id);
   1201         drop_template (t);
   1202         break;
   1203       }
   1204       if (loaded_paivana_templates >= MAX_PAIVANA_TEMPLATES)
   1205       {
   1206         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1207                     "Merchant offers more than %u Paivana templates;"
   1208                     " refusing startup to bound regex work and retained"
   1209                     " contract memory\n",
   1210                     (unsigned int) MAX_PAIVANA_TEMPLATES);
   1211         PH_global_ret = EXIT_NOTCONFIGURED;
   1212         GNUNET_SCHEDULER_shutdown ();
   1213         return;
   1214       }
   1215       contract_size = json_dumpb (contract,
   1216                                   NULL,
   1217                                   0,
   1218                                   JSON_COMPACT);
   1219       if (contract_size > MAX_TEMPLATE_CONTRACT_SIZE)
   1220       {
   1221         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1222                     "Template %s has a %llu-byte contract, exceeding the"
   1223                     " %u-byte startup and retained-memory limit\n",
   1224                     t->template_id,
   1225                     (unsigned long long) contract_size,
   1226                     (unsigned int) MAX_TEMPLATE_CONTRACT_SIZE);
   1227         PH_global_ret = EXIT_NOTCONFIGURED;
   1228         GNUNET_SCHEDULER_shutdown ();
   1229         return;
   1230       }
   1231       if (! parse_template (t,
   1232                             contract))
   1233       {
   1234         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1235                     "Failed to parse template %s, refusing to start\n",
   1236                     t->template_id);
   1237         PH_global_ret = EXIT_FAILURE;
   1238         GNUNET_SCHEDULER_shutdown ();
   1239         return;
   1240       }
   1241       loaded_paivana_templates++;
   1242       break;
   1243     }
   1244   default:
   1245     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1246                 "Failed to load template %s from backend"
   1247                 " (HTTP status %u), refusing to start\n",
   1248                 t->template_id,
   1249                 tgr->hr.http_status);
   1250     PH_global_ret = EXIT_FAILURE;
   1251     GNUNET_SCHEDULER_shutdown ();
   1252     return;
   1253   }
   1254   start_template_fetches ();
   1255   finish_template_loading ();
   1256 }
   1257 
   1258 
   1259 /** Start waiting detail requests until the eight-request window is full. */
   1260 static void
   1261 start_template_fetches (void)
   1262 {
   1263   while ( (active_template_fetches < MAX_TEMPLATE_FETCHES) &&
   1264           (0 != pending_template_fetches) )
   1265   {
   1266     struct Template *t = next_template_fetch;
   1267     enum TALER_ErrorCode ec;
   1268 
   1269     GNUNET_assert (NULL != t);
   1270     next_template_fetch = t->next;
   1271     GNUNET_assert (TLS_WAITING == t->load_state);
   1272     t->gt = TALER_MERCHANT_get_private_template_create (
   1273       PH_merchant_ctx,
   1274       PH_merchant_internal_url,
   1275       t->template_id);
   1276     if (NULL == t->gt)
   1277     {
   1278       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1279                   "Could not allocate merchant request for template %s\n",
   1280                   t->template_id);
   1281       PH_global_ret = EXIT_FAILURE;
   1282       GNUNET_SCHEDULER_shutdown ();
   1283       return;
   1284     }
   1285     t->load_state = TLS_ACTIVE;
   1286     pending_template_fetches--;
   1287     active_template_fetches++;
   1288     ec = TALER_MERCHANT_get_private_template_start (t->gt,
   1289                                                     &setup_template,
   1290                                                     t);
   1291     if (TALER_EC_NONE != ec)
   1292     {
   1293       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1294                   "Could not start merchant request for template %s: %d\n",
   1295                   t->template_id,
   1296                   (int) ec);
   1297       TALER_MERCHANT_get_private_template_cancel (t->gt);
   1298       t->gt = NULL;
   1299       t->load_state = TLS_DONE;
   1300       active_template_fetches--;
   1301       PH_global_ret = EXIT_FAILURE;
   1302       GNUNET_SCHEDULER_shutdown ();
   1303       return;
   1304     }
   1305   }
   1306 }
   1307 
   1308 
   1309 /**
   1310  * Callback for a GET /private/templates request.
   1311  *
   1312  * @param cls unused
   1313  * @param tgr response details
   1314  */
   1315 static void
   1316 check_templates (
   1317   void *cls,
   1318   const struct TALER_MERCHANT_GetPrivateTemplatesResponse *tgr)
   1319 {
   1320   (void) cls;
   1321   gpt = NULL;
   1322   /* The merchant client library enforces the same 1024-entry ceiling before
   1323      filling details.ok.  Preserve Paivana's own operator-facing diagnosis by
   1324      recognizing that parse-failure shape in the raw reply; without it the
   1325      callback reports only status zero/error 10 and the configured bound is
   1326      invisible. */
   1327   if ( (0 == tgr->hr.http_status) &&
   1328        (NULL != tgr->hr.reply) )
   1329   {
   1330     const json_t *templates = json_object_get (tgr->hr.reply,
   1331                                                "templates");
   1332 
   1333     if ( json_is_array (templates) &&
   1334          (json_array_size (templates) > MAX_DISCOVERED_TEMPLATES) )
   1335     {
   1336       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1337                   "Merchant returned %llu template IDs, exceeding the"
   1338                   " %u-entry startup bound\n",
   1339                   (unsigned long long) json_array_size (templates),
   1340                   (unsigned int) MAX_DISCOVERED_TEMPLATES);
   1341       PH_global_ret = EXIT_NOTCONFIGURED;
   1342       GNUNET_SCHEDULER_shutdown ();
   1343       return;
   1344     }
   1345   }
   1346   switch (tgr->hr.http_status)
   1347   {
   1348   case MHD_HTTP_OK:
   1349     break;
   1350   case MHD_HTTP_UNAUTHORIZED:
   1351     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1352                 "Access to templates unauthorized: %s\n",
   1353                 TALER_ErrorCode_get_hint (tgr->hr.ec));
   1354     PH_global_ret = EXIT_FAILURE;
   1355     GNUNET_SCHEDULER_shutdown ();
   1356     return;
   1357   default:
   1358     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1359                 "Unexpected HTTP status code %u on GET /private/templates (%d)\n",
   1360                 tgr->hr.http_status,
   1361                 (int) tgr->hr.ec);
   1362     PH_global_ret = EXIT_FAILURE;
   1363     GNUNET_SCHEDULER_shutdown ();
   1364     return;
   1365   }
   1366   if (0 == tgr->details.ok.templates_length)
   1367   {
   1368     templates_ready ();
   1369     return;
   1370   }
   1371   if (tgr->details.ok.templates_length > MAX_DISCOVERED_TEMPLATES)
   1372   {
   1373     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1374                 "Merchant returned %u template IDs, exceeding the %u-entry"
   1375                 " startup bound\n",
   1376                 tgr->details.ok.templates_length,
   1377                 (unsigned int) MAX_DISCOVERED_TEMPLATES);
   1378     PH_global_ret = EXIT_NOTCONFIGURED;
   1379     GNUNET_SCHEDULER_shutdown ();
   1380     return;
   1381   }
   1382 
   1383   for (unsigned int i = 0; i<tgr->details.ok.templates_length; i++)
   1384   {
   1385     const struct TALER_MERCHANT_GetPrivateTemplatesTemplateEntry *te
   1386       = &tgr->details.ok.templates[i];
   1387     struct Template *t;
   1388 
   1389     t = GNUNET_new (struct Template);
   1390     t->template_id = GNUNET_strdup (te->template_id);
   1391     t->max_pickup_delay = GNUNET_TIME_UNIT_FOREVER_REL;
   1392     t->load_state = TLS_WAITING;
   1393     GNUNET_CONTAINER_DLL_insert_tail (t_head,
   1394                                       t_tail,
   1395                                       t);
   1396   }
   1397   pending_template_fetches = tgr->details.ok.templates_length;
   1398   next_template_fetch = t_head;
   1399   start_template_fetches ();
   1400 }
   1401 
   1402 
   1403 void
   1404 PAIVANA_HTTPD_load_templates ()
   1405 {
   1406   GNUNET_assert (NULL == load_timeout_task);
   1407   load_timeout_task
   1408     = GNUNET_SCHEDULER_add_delayed (TEMPLATE_LOAD_TIMEOUT,
   1409                                     &load_timeout,
   1410                                     NULL);
   1411   gpt = TALER_MERCHANT_get_private_templates_create (PH_merchant_ctx,
   1412                                                      PH_merchant_internal_url);
   1413   if (NULL == gpt)
   1414   {
   1415     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1416                 "Could not allocate merchant template-list request\n");
   1417     PH_global_ret = EXIT_FAILURE;
   1418     GNUNET_SCHEDULER_shutdown ();
   1419     return;
   1420   }
   1421   {
   1422     enum TALER_ErrorCode ec;
   1423 
   1424     ec = TALER_MERCHANT_get_private_templates_start (gpt,
   1425                                                      &check_templates,
   1426                                                      NULL);
   1427     if (TALER_EC_NONE != ec)
   1428     {
   1429       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1430                   "Could not start merchant template-list request: %d\n",
   1431                   (int) ec);
   1432       TALER_MERCHANT_get_private_templates_cancel (gpt);
   1433       gpt = NULL;
   1434       PH_global_ret = EXIT_FAILURE;
   1435       GNUNET_SCHEDULER_shutdown ();
   1436     }
   1437   }
   1438 }
   1439 
   1440 
   1441 enum GNUNET_GenericReturnValue
   1442 PAIVANA_HTTPD_search_templates (struct MHD_Connection *connection,
   1443                                 const char *website)
   1444 {
   1445   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
   1446               "Searching templates for `%s'\n",
   1447               website);
   1448   if (PH_MAX_URL_LENGTH < strlen (website))
   1449   {
   1450     enum MHD_Result ret;
   1451 
   1452     /* Refuse rather than returning #GNUNET_SYSERR: that would mean
   1453        "no paywall applies" and hand the request to the upstream for
   1454        free. */
   1455     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
   1456                 "Refusing to match templates against %llu byte URL\n",
   1457                 (unsigned long long) strlen (website));
   1458     ret = TALER_MHD_reply_with_error (
   1459       connection,
   1460       MHD_HTTP_URI_TOO_LONG,
   1461       TALER_EC_GENERIC_URI_TOO_LONG,
   1462       NULL);
   1463     return (MHD_YES == ret) ? GNUNET_OK : GNUNET_NO;
   1464   }
   1465   for (struct Template *t = t_head; NULL != t; t = t->next)
   1466   {
   1467     struct MHD_Response *redirect;
   1468     enum MHD_Result ret;
   1469     struct GNUNET_Buffer buf = { 0 };
   1470     char *enc = NULL;
   1471     char *url;
   1472 
   1473     if (NULL != t->regex)
   1474     {
   1475       int rc;
   1476 
   1477       rc = regexec (&t->ex,
   1478                     website,
   1479                     0, NULL,
   1480                     0);
   1481       if (REG_NOMATCH == rc)
   1482       {
   1483         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
   1484                     "Request for %s did not match template %s\n",
   1485                     website,
   1486                     t->template_id);
   1487         continue;
   1488       }
   1489       if (0 != rc)
   1490       {
   1491         char errbuf[128];
   1492 
   1493         /* Not "did not match": regexec(3) also reports REG_ESPACE,
   1494            whose likelihood is a function of the merchant's pattern and
   1495            the client's URL.  Taking the `continue' would drop this
   1496            template from consideration, and if it were the last one the
   1497            caller reads #GNUNET_SYSERR as "no paywall applies" and
   1498            serves the page for free.  Fail closed instead. */
   1499         GNUNET_break (0);
   1500         (void) regerror (rc,
   1501                          &t->ex,
   1502                          errbuf,
   1503                          sizeof (errbuf));
   1504         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
   1505                     "Failed to match template %s against %s: %s\n",
   1506                     t->template_id,
   1507                     website,
   1508                     errbuf);
   1509         ret = TALER_MHD_reply_with_error (
   1510           connection,
   1511           MHD_HTTP_INTERNAL_SERVER_ERROR,
   1512           TALER_EC_GENERIC_INTERNAL_INVARIANT_FAILURE,
   1513           errbuf);
   1514         return (MHD_YES == ret) ? GNUNET_OK : GNUNET_NO;
   1515       }
   1516     }
   1517 
   1518     if (! PAIVANA_HTTPD_get_base_url (connection,
   1519                                       &buf))
   1520     {
   1521       GNUNET_break (0);
   1522       GNUNET_buffer_clear (&buf);
   1523       ret = TALER_MHD_reply_with_error (
   1524         connection,
   1525         MHD_HTTP_BAD_REQUEST,
   1526         TALER_EC_GENERIC_HTTP_HEADERS_MALFORMED,
   1527         "Host or X-Forwarded-Host required");
   1528       return (MHD_YES == ret) ? GNUNET_OK : GNUNET_NO;
   1529     }
   1530     (void) GNUNET_STRINGS_base64url_encode (website,
   1531                                             strlen (website),
   1532                                             &enc);
   1533     GNUNET_buffer_write_str (&buf,
   1534                              "/.well-known/paivana/templates/");
   1535     GNUNET_buffer_write_str (&buf,
   1536                              t->template_id);
   1537     GNUNET_buffer_write_str (&buf,
   1538                              "#");
   1539     GNUNET_buffer_write_str (&buf,
   1540                              enc);
   1541     GNUNET_free (enc);
   1542     url = GNUNET_buffer_reap_str (&buf);
   1543     redirect = MHD_create_response_from_buffer_static (0,
   1544                                                        NULL);
   1545     GNUNET_assert (NULL != redirect);
   1546     GNUNET_break (MHD_YES ==
   1547                   MHD_add_response_header (redirect,
   1548                                            MHD_HTTP_HEADER_LOCATION,
   1549                                            url));
   1550     /* Both the Location and the fragment are built from the base URL,
   1551        which -- unless BASE_URL pins it -- comes out of the forwarding
   1552        headers.  A cache sitting between the terminating proxy and us
   1553        would otherwise serve one client's redirect to another virtual
   1554        host (RFC 9110 section 12.5.5). */
   1555     GNUNET_break (MHD_YES ==
   1556                   MHD_add_response_header (redirect,
   1557                                            MHD_HTTP_HEADER_VARY,
   1558                                            (NULL != PH_base_url)
   1559                                            ? "Cookie"
   1560                                            : "Cookie, "
   1561                                            MHD_HTTP_HEADER_FORWARDED ", "
   1562                                            PH_HEADER_X_FORWARDED_PROTO ", "
   1563                                            PH_HEADER_X_FORWARDED_HOST ", "
   1564                                            PH_HEADER_X_FORWARDED_PORT));
   1565     GNUNET_break (MHD_YES ==
   1566                   MHD_add_response_header (redirect,
   1567                                            MHD_HTTP_HEADER_CACHE_CONTROL,
   1568                                            "public, max-age=60"));
   1569     GNUNET_free (url);
   1570     ret = MHD_queue_response (connection,
   1571                               MHD_HTTP_FOUND,
   1572                               redirect);
   1573     MHD_destroy_response (redirect);
   1574     return (MHD_YES == ret) ? GNUNET_OK : GNUNET_NO;
   1575   }
   1576   return GNUNET_SYSERR;
   1577 }
   1578 
   1579 
   1580 /**
   1581  * Return the paywall page for the given @a template.
   1582  *
   1583  * @param connection request to search paywall response for
   1584  * @param template template to return paywall page for
   1585  * @return MHD status code
   1586  */
   1587 enum MHD_Result
   1588 PAIVANA_HTTPD_return_template (struct MHD_Connection *connection,
   1589                                const char *template)
   1590 {
   1591   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
   1592               "Searching template `%s'\n",
   1593               template);
   1594   for (struct Template *t = t_head; NULL != t; t = t->next)
   1595   {
   1596     if (0 == strcmp (template,
   1597                      t->template_id))
   1598       return load_paywall (connection,
   1599                            t);
   1600   }
   1601   /* No GNUNET_break_op() here: the ID is whatever the client put in the
   1602      path, this runs before any payment, and a stale bookmark or a
   1603      crawler would otherwise write an ERROR-level "Assertion failed" per
   1604      request.  A 404 is the whole answer. */
   1605   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
   1606               "No template `%s', returning 404\n",
   1607               template);
   1608   return TALER_MHD_reply_with_error (connection,
   1609                                      MHD_HTTP_NOT_FOUND,
   1610                                      TALER_EC_PAIVANA_TEMPLATE_UNKNOWN,
   1611                                      template);
   1612 }
   1613 
   1614 
   1615 /**
   1616  * Unload all of the template state.
   1617  */
   1618 void
   1619 PAIVANA_HTTPD_unload_templates ()
   1620 {
   1621   if (NULL != load_timeout_task)
   1622   {
   1623     GNUNET_SCHEDULER_cancel (load_timeout_task);
   1624     load_timeout_task = NULL;
   1625   }
   1626   while (NULL != rce_head)
   1627   {
   1628     struct ResponseCacheEntry *rce = rce_head;
   1629 
   1630     GNUNET_assert (rce_length > 0);
   1631     rce_length--;
   1632     GNUNET_CONTAINER_DLL_remove (rce_head,
   1633                                  rce_tail,
   1634                                  rce);
   1635     MHD_destroy_response (rce->paywall);
   1636     GNUNET_free (rce->lang);
   1637     GNUNET_free (rce);
   1638   }
   1639   while (NULL != t_head)
   1640   {
   1641     struct Template *t = t_head;
   1642 
   1643     GNUNET_CONTAINER_DLL_remove (t_head,
   1644                                  t_tail,
   1645                                  t);
   1646     if (NULL != t->gt)
   1647       TALER_MERCHANT_get_private_template_cancel (t->gt);
   1648     if (NULL != t->regex)
   1649     {
   1650       regfree (&t->ex);
   1651       GNUNET_free (t->regex);
   1652     }
   1653     GNUNET_free (t->template_id);
   1654     GNUNET_free (t->summary);
   1655     json_decref (t->choices);
   1656     GNUNET_free (t);
   1657   }
   1658   for (unsigned int i = 0; i < paywall_languages_length; i++)
   1659     GNUNET_free (paywall_languages[i]);
   1660   paywall_languages_length = 0;
   1661   pending_template_fetches = 0;
   1662   active_template_fetches = 0;
   1663   loaded_paivana_templates = 0;
   1664   next_template_fetch = NULL;
   1665   if (NULL != gpt)
   1666   {
   1667     TALER_MERCHANT_get_private_templates_cancel (gpt);
   1668     gpt = NULL;
   1669   }
   1670   GNUNET_free (paywall_csp);
   1671 }