use external helper for conversion also for KYCAID

4 files changed, 215 insertions, 236 deletions

diff --git a/src/kyclogic/Makefile.am b/src/kyclogic/Makefile.am
index 20430a4ea..75ea13b6f 100644
--- a/src/kyclogic/Makefile.am
+++ b/src/kyclogic/Makefile.am
@@ -20,6 +20,7 @@ EXTRA_DIST = \
   persona-sample-reply.json
 
 bin_SCRIPTS = \
+  taler-exchange-kyc-kycaid-converter.sh \
   taler-exchange-kyc-persona-converter.sh
 
 lib_LTLIBRARIES = \
diff --git a/src/kyclogic/kyclogic-kycaid.conf b/src/kyclogic/kyclogic-kycaid.conf
index 0e1fe96ef..753fb689d 100644
--- a/src/kyclogic/kyclogic-kycaid.conf
+++ b/src/kyclogic/kyclogic-kycaid.conf
@@ -12,6 +12,10 @@ PROVIDED_CHECKS = EXAMPLE_DO_NOT_USE
 # How long is the KYC check valid?
 KYC_KYCAID_VALIDITY = forever
 
+# Program that converts Persona KYC data into the
+# GNU Taler format.
+KYC_KYCAID_CONVERTER_HELPER = taler-exchange-kyc-kycaid-converter.sh
+
 # Authentication token to use.
 KYC_KYCAID_AUTH_TOKEN = XXX
 
diff --git a/src/kyclogic/plugin_kyclogic_kycaid.c b/src/kyclogic/plugin_kyclogic_kycaid.c
index 3273a51f1..5dad94d91 100644
--- a/src/kyclogic/plugin_kyclogic_kycaid.c
+++ b/src/kyclogic/plugin_kyclogic_kycaid.c
@@ -88,6 +88,12 @@ struct TALER_KYCLOGIC_ProviderDetails
   char *form_id;
 
   /**
+   * Helper binary to convert attributes returned by
+   * KYCAID into our internal format.
+   */
+  char *conversion_helper;
+
+  /**
    * Validity time for a successful KYC process.
    */
   struct GNUNET_TIME_Relative validity;
@@ -216,6 +222,12 @@ struct TALER_KYCLOGIC_WebhookHandle
   struct PluginState *ps;
 
   /**
+   * Handle to helper process to extract attributes
+   * we care about.
+   */
+  struct TALER_JSON_ExternalConversion *econ;
+
+  /**
    * Our configuration details.
    */
   const struct TALER_KYCLOGIC_ProviderDetails *pd;
@@ -226,6 +238,11 @@ struct TALER_KYCLOGIC_WebhookHandle
   struct MHD_Connection *connection;
 
   /**
+   * JSON response we got back, or NULL for none.
+   */
+  json_t *json_response;
+
+  /**
    * Verification ID from the service.
    */
   char *verification_id;
@@ -262,6 +279,11 @@ struct TALER_KYCLOGIC_WebhookHandle
   uint64_t process_row;
 
   /**
+   * HTTP response code we got from KYCAID.
+   */
+  unsigned int kycaid_response_code;
+
+  /**
    * HTTP response code to return asynchronously.
    */
   unsigned int response_code;
@@ -277,6 +299,7 @@ static void
 kycaid_unload_configuration (struct TALER_KYCLOGIC_ProviderDetails *pd)
 {
   curl_slist_free_all (pd->slist);
+  GNUNET_free (pd->conversion_helper);
   GNUNET_free (pd->auth_token);
   GNUNET_free (pd->form_id);
   GNUNET_free (pd->section);
@@ -337,6 +360,18 @@ kycaid_load_configuration (void *cls,
     kycaid_unload_configuration (pd);
     return NULL;
   }
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_filename (ps->cfg,
+                                               provider_section_name,
+                                               "KYC_KYCAID_CONVERTER_HELPER",
+                                               &pd->conversion_helper))
+  {
+    GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+                               provider_section_name,
+                               "KYC_KYCAID_CONVERTER_HELPER");
+    kycaid_unload_configuration (pd);
+    return NULL;
+  }
   {
     char *auth;
 
@@ -695,11 +730,21 @@ kycaid_webhook_cancel (struct TALER_KYCLOGIC_WebhookHandle *wh)
     GNUNET_SCHEDULER_cancel (wh->task);
     wh->task = NULL;
   }
+  if (NULL != wh->econ)
+  {
+    TALER_JSON_external_conversion_stop (wh->econ);
+    wh->econ = NULL;
+  }
   if (NULL != wh->job)
   {
     GNUNET_CURL_job_cancel (wh->job);
     wh->job = NULL;
   }
+  if (NULL != wh->json_response)
+  {
+    json_decref (wh->json_response);
+    wh->json_response = NULL;
+  }
   GNUNET_free (wh->verification_id);
   GNUNET_free (wh->applicant_id);
   GNUNET_free (wh->url);
@@ -751,6 +796,97 @@ log_failure (json_t *verifications)
 
 
 /**
+ * Type of a callback that receives a JSON @a result.
+ *
+ * @param cls closure our `struct TALER_KYCLOGIC_WebhookHandle *`
+ * @param status_type how did the process die
+ * @param code termination status code from the process
+ * @param result converted attribute data, NULL on failure
+ */
+static void
+webhook_conversion_cb (void *cls,
+                       enum GNUNET_OS_ProcessStatusType status_type,
+                       unsigned long code,
+                       const json_t *result)
+{
+  struct TALER_KYCLOGIC_WebhookHandle *wh = cls;
+  struct GNUNET_TIME_Absolute expiration;
+  struct MHD_Response *resp;
+
+  wh->econ = NULL;
+  if ( (0 == code) ||
+       (NULL == result) )
+  {
+    /* No result, but *our helper* was OK => bad input */
+    GNUNET_break_op (0);
+    json_dumpf (wh->json_response,
+                stderr,
+                JSON_INDENT (2));
+    resp = TALER_MHD_MAKE_JSON_PACK (
+      GNUNET_JSON_pack_uint64 ("kycaid_http_status",
+                               wh->kycaid_response_code),
+      GNUNET_JSON_pack_object_incref ("kycaid_body",
+                                      (json_t *) wh->json_response));
+    wh->cb (wh->cb_cls,
+            wh->process_row,
+            &wh->h_payto,
+            wh->pd->section,
+            wh->applicant_id,
+            wh->verification_id,
+            TALER_KYCLOGIC_STATUS_PROVIDER_FAILED,
+            GNUNET_TIME_UNIT_ZERO_ABS, /* expiration */
+            NULL,
+            MHD_HTTP_BAD_GATEWAY,
+            resp);
+    kycaid_webhook_cancel (wh);
+    return;
+  }
+  if (NULL == result)
+  {
+    /* Failure in our helper */
+    GNUNET_break (0);
+    json_dumpf (wh->json_response,
+                stderr,
+                JSON_INDENT (2));
+    resp = TALER_MHD_MAKE_JSON_PACK (
+      GNUNET_JSON_pack_uint64 ("kycaid_http_status",
+                               wh->kycaid_response_code),
+      GNUNET_JSON_pack_object_incref ("kycaid_body",
+                                      (json_t *) wh->json_response));
+    wh->cb (wh->cb_cls,
+            wh->process_row,
+            &wh->h_payto,
+            wh->pd->section,
+            wh->applicant_id,
+            wh->verification_id,
+            TALER_KYCLOGIC_STATUS_PROVIDER_FAILED,
+            GNUNET_TIME_UNIT_ZERO_ABS, /* expiration */
+            NULL,
+            MHD_HTTP_BAD_GATEWAY,
+            resp);
+    kycaid_webhook_cancel (wh);
+    return;
+  }
+  expiration = GNUNET_TIME_relative_to_absolute (wh->pd->validity);
+  resp = MHD_create_response_from_buffer (0,
+                                          "",
+                                          MHD_RESPMEM_PERSISTENT);
+  wh->cb (wh->cb_cls,
+          wh->process_row,
+          &wh->h_payto,
+          wh->pd->section,
+          wh->applicant_id,
+          wh->verification_id,
+          TALER_KYCLOGIC_STATUS_SUCCESS,
+          expiration,
+          result,
+          MHD_HTTP_NO_CONTENT,
+          resp);
+  kycaid_webhook_cancel (wh);
+}
+
+
+/**
  * Function called when we're done processing the
  * HTTP "/applicants/{verification_id}" request.
  *
@@ -768,253 +904,30 @@ handle_webhook_finished (void *cls,
   struct MHD_Response *resp;
 
   wh->job = NULL;
+  wh->kycaid_response_code = response_code;
+  wh->json_response = json_incref ((json_t *) j);
   switch (response_code)
   {
   case MHD_HTTP_OK:
     {
-      const char *type;
       const char *profile_status;
-      const char *first_name = NULL;
-      const char *last_name = NULL;
-      const char *middle_name = NULL;
-      const char *dob = NULL;
-      const char *residence_country = NULL;
-      const char *gender = NULL;
-      bool pep = false;
-      bool no_pep = false;
-      const char *company_name = NULL;
-      const char *business_activity_id = NULL;
-      const char *registration_country = NULL;
-      const char *email = NULL;
-      const char *phone = NULL;
-      json_t *addresses = NULL;
-      json_t *documents = NULL;
-      struct GNUNET_JSON_Specification spec[] = {
-        GNUNET_JSON_spec_string ("type",
-                                 &type),
-        GNUNET_JSON_spec_string ("profile_status",
-                                 &profile_status), /* valid, invalid, pending */
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("email",
-                                   &email),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("phone",
-                                   &phone),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_json ("addresses",
-                                 &addresses),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_json ("documents",
-                                 &documents),
-          NULL),
-        GNUNET_JSON_spec_end ()
-      };
-      struct GNUNET_JSON_Specification bspec[] = {
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("company_name",
-                                   &company_name),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("business_activity_id",
-                                   &business_activity_id),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("registration_country",
-                                   &registration_country),
-          NULL),
-        GNUNET_JSON_spec_end ()
-      };
-      struct GNUNET_JSON_Specification pspec[] = {
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("first_name",
-                                   &first_name),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("middle_name",
-                                   &middle_name),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("last_name",
-                                   &last_name),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("dob",
-                                   &dob),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("residence_country",
-                                   &residence_country),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_string ("gender",
-                                   &gender),
-          NULL),
-        GNUNET_JSON_spec_mark_optional (
-          GNUNET_JSON_spec_bool ("pep",
-                                 &pep),
-          &no_pep),
-        GNUNET_JSON_spec_end ()
-      };
-      struct GNUNET_JSON_Specification *ispec = NULL;
-      struct GNUNET_TIME_Absolute expiration;
-      bool no_parse;
-      enum TALER_KYCLOGIC_KycUserType ut;
-
-      no_parse = (GNUNET_OK !=
-                  GNUNET_JSON_parse (j,
-                                     spec,
-                                     NULL, NULL));
-      if (! no_parse)
-      {
-        ut = (0 == strcasecmp ("person",
-                               type))
-          ? TALER_KYCLOGIC_KYC_UT_INDIVIDUAL
-          : TALER_KYCLOGIC_KYC_UT_BUSINESS;
-        ispec = (ut == TALER_KYCLOGIC_KYC_UT_INDIVIDUAL)
-          ? pspec
-          : bspec;
-        no_parse = (GNUNET_OK !=
-                    GNUNET_JSON_parse (j,
-                                       ispec,
-                                       NULL, NULL));
-      }
-      if (no_parse)
-      {
-        GNUNET_break_op (0);
-        json_dumpf (j,
-                    stderr,
-                    JSON_INDENT (2));
-        resp = TALER_MHD_MAKE_JSON_PACK (
-          GNUNET_JSON_pack_uint64 ("kycaid_http_status",
-                                   response_code),
-          GNUNET_JSON_pack_object_incref ("kycaid_body",
-                                          (json_t *) j));
-        wh->cb (wh->cb_cls,
-                wh->process_row,
-                &wh->h_payto,
-                wh->pd->section,
-                wh->applicant_id,
-                wh->verification_id,
-                TALER_KYCLOGIC_STATUS_PROVIDER_FAILED,
-                GNUNET_TIME_UNIT_ZERO_ABS, /* expiration */
-                NULL,
-                MHD_HTTP_BAD_GATEWAY,
-                resp);
-        break;
-      }
-      if (0 == strcasecmp ("valid",
-                           profile_status))
-      {
-        log_failure (json_object_get (j,
-                                      "decline_reasons"));
-      }
-      resp = MHD_create_response_from_buffer (0,
-                                              "",
-                                              MHD_RESPMEM_PERSISTENT);
-      if (0 == strcasecmp ("valid",
+
+      profile_status = json_string_value (
+        json_object_get (
+          j,
+          "profile_status"));
+      if (0 != strcasecmp ("valid",
                            profile_status))
       {
-        json_t *attr;
-
-        if (ut == TALER_KYCLOGIC_KYC_UT_INDIVIDUAL)
-        {
-          char *name = NULL;
-
-          if ( (NULL != last_name) ||
-               (NULL != first_name) ||
-               (NULL != middle_name) )
-          {
-            GNUNET_asprintf (&name,
-                             "%s, %s %s",
-                             (NULL != last_name)
-                             ? last_name
-                             : "",
-                             (NULL != first_name)
-                             ? first_name
-                             : "",
-                             (NULL != middle_name)
-                             ? middle_name
-                             : "");
-          }
-          attr = GNUNET_JSON_PACK (
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_BIRTHDATE,
-                dob)),
-            GNUNET_JSON_pack_allow_null (
-              no_pep
-              ? GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_PEP,
-                NULL)
-              : GNUNET_JSON_pack_bool (
-                TALER_ATTRIBUTE_PEP,
-                pep)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_FULL_NAME,
-                name)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_PHONE,
-                phone)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_EMAIL,
-                email)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_RESIDENCES,
-                residence_country))
-            );
-          GNUNET_free (name);
-        }
-        else
-        {
-          attr = GNUNET_JSON_PACK (
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_COMPANY_NAME,
-                company_name)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_PHONE,
-                phone)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_EMAIL,
-                email)),
-            GNUNET_JSON_pack_allow_null (
-              GNUNET_JSON_pack_string (
-                TALER_ATTRIBUTE_REGISTRATION_COUNTRY,
-                residence_country))
-            );
-        }
-        // FIXME: do something about addresses & documents!
-        expiration = GNUNET_TIME_relative_to_absolute (wh->pd->validity);
-        wh->cb (wh->cb_cls,
-                wh->process_row,
-                &wh->h_payto,
-                wh->pd->section,
-                wh->applicant_id,
-                wh->verification_id,
-                TALER_KYCLOGIC_STATUS_SUCCESS,
-                expiration,
-                attr,
-                MHD_HTTP_NO_CONTENT,
-                resp);
-        json_decref (attr);
-      }
-      else
-      {
         enum TALER_KYCLOGIC_KycStatus ks;
 
         ks = (0 == strcasecmp ("pending",
                                profile_status))
           ? TALER_KYCLOGIC_STATUS_PENDING
           : TALER_KYCLOGIC_STATUS_USER_ABORTED;
+        resp = MHD_create_response_from_buffer (0,
+                                                "",
+                                                MHD_RESPMEM_PERSISTENT);
         wh->cb (wh->cb_cls,
                 wh->process_row,
                 &wh->h_payto,
@@ -1026,9 +939,15 @@ handle_webhook_finished (void *cls,
                 NULL,
                 MHD_HTTP_NO_CONTENT,
                 resp);
+        break;
       }
-      GNUNET_JSON_parse_free (ispec);
-      GNUNET_JSON_parse_free (spec);
+      wh->econ = TALER_JSON_external_conversion_start (j,
+                                                       &webhook_conversion_cb,
+                                                       wh,
+                                                       wh->pd->conversion_helper,
+                                                       wh->pd->conversion_helper,
+                                                       NULL);
+      return;
     }
     break;
   case MHD_HTTP_BAD_REQUEST:
diff --git a/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh b/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh
new file mode 100644
index 000000000..6e261eaf4
--- /dev/null
+++ b/src/kyclogic/taler-exchange-kyc-kycaid-converter.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+# This file is in the public domain.
+#
+# This code converts (some of) the JSON output from KYCAID into the GNU Taler
+# specific KYC attribute data (again in JSON format).  We may need to download
+# and inline file data in the process, for authorization pass "-a" with the
+# respective bearer token.
+#
+
+# Die if anything goes wrong.
+set -eu
+
+# Parse command-line options
+while getopts ':a:' OPTION; do
+    case "$OPTION" in
+        a)
+            TOKEN="$OPTARG"
+            ;;
+        ?)
+        echo "Unrecognized command line option"
+        exit 1
+        ;;
+    esac
+done
+
+# First, extract everything from stdin.
+J=$(jq '{"type":.type,"email":.email,"phone":.phone,"first_name":.first_name,"name-middle":.middle_name,"last_name":.last_name,"dob":.dob,"residence_country":.residence_country,"gender":.gender,"pep":.pep,"addresses":.addresses,"documents":.documents,"company_name":.company_name,"business_activity_id":.business_activity_id,"registration_country":.registration_country}')
+
+# TODO:
+# log_failure (json_object_get (j, "decline_reasons"));
+
+TYPE=$(echo "$J" | jq -r '.person')
+
+if [ "person" = "${TYPE}" ]
+then
+
+  # Next, combine some fields into larger values.
+  FULLNAME=$(echo "$J" | jq -r '[.first_name,.middle_name,.last_name]|join(" ")')
+#  STREET=$(echo $J | jq -r '[."street-1",."street-2"]|join(" ")')
+#  CITY=$(echo $J | jq -r '[.postcode,.city,."address-subdivision,.cc"]|join(" ")')
+
+  # Combine into final result for individual.
+  # FIXME: does jq tolerate 'pep = NULL' here?
+  echo "$J" | jq \
+    --arg full_name "${FULLNAME}" \
+    '{$full_name,"birthdate":.dob,"pep":.pep,"phone":."phone","email",.email,"residences":.residence_country}'
+
+else
+  # Combine into final result for business.
+  echo "$J" | jq \
+    --arg full_name "${FULLNAME}" \
+    '{"company_name":.company_name,"phone":.phone,"email":.email,"registration_country":.registration_country}'
+fi
+
+exit 0