kyc spec work

1 files changed, 205 insertions, 40 deletions

diff --git a/design-documents/023-taler-kyc.rst b/design-documents/023-taler-kyc.rst
index f47e98a4..85478d8e 100644
--- a/design-documents/023-taler-kyc.rst
+++ b/design-documents/023-taler-kyc.rst
@@ -431,12 +431,17 @@ providers, one per configuration section:
     # (for any reason, e.g. provider or form fail to
     # satisfy constraints or provider signals user error)
     # Usually should point to a measure that requests
-    # AML staff to investigate.
+    # AML staff to investigate.  The fallback measure
+    # context always includes the reasons for the
+    # failure.
     FALLBACK = MEASURE_NAME
 
 The list of possible FORM names is fixed in the SPA
 for a particular exchange release.
 
+The outcome of *any* check should always be uploaded encrypted into the
+``kyc_attributes`` table.  It MUST include an ``expiration_time``.
+
 
 Configuration of legitimization requirement triggers
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -454,8 +459,21 @@ configuration section:
     # or WALLET-BALANCE.
     OPERATION_TYPE = WITHDRAW
 
-    # Required measure to be performed.
-    REQUIRED_MEASURE = SWISSNESS
+    # Next measures to be performed. The SPA should
+    # display *all* of these measures to the user.
+    # (they have a choice of either which ones, or in
+    # which order they are to be performed).
+    NEXT_MEASURES = SWISSNESS KYB
+
+    # Context for each of the above measures, optional.
+    MEASURE_CONTEXT_$NAME = CONTEXT
+
+    # AND if all REQUIRED_MEASURES will eventually need
+    # to be satisfied, OR if the user has a choice between
+    # them. Not actually enforced by the exchange, but
+    # primarily used to inform the user whether this is
+    # an "and" or "or".
+    COMBINATOR = AND|OR
 
     # Threshold amount above which the legitimization is
     # triggered.  The total must be exceeded in the given
@@ -489,7 +507,9 @@ AML programs are helper programs that can:
   ``[kyc-provider-]`` configuration section
   (but may also include FORM field names).
 * Process an input JSON object with context and
-  attributes into an *outcome*.
+  attributes into an *outcome*.  This is the
+  default behavior if no command-line switches
+  are provided.
 
 AML programs are listed in the configuration file,
 one program per section:
@@ -506,10 +526,12 @@ one program per section:
 
     # **original** measure to take if COMMAND fails
     # Usually points to a measure that asks AML staff
-    # to contact the systems administrator.
+    # to contact the systems administrator. The fallback measure
+    # context always includes the reasons for the
+    # failure.
     FALLBACK = MEASURE_NAME
 
-The JSON input of AML programs consist of two parts:
+The JSON input of an AML program consists of three parts:
 
 * "context": JSON object that was provided as
   part of the *measure*.  This JSON object is
@@ -524,12 +546,15 @@ The JSON input of AML programs consist of two parts:
   names and the values must be strings representing
   the data. In the case of file uploads, the data
   MUST be base64-encoded.
+* "history": JSON array with the results of historic
+  data collected about the user.
 
-The outputs of AML programs must be JSON objects which must state:
+The output of an AML programs must be JSON objects which must state:
 
 * outcome: what to do with the client's account
 * expiration_date: when does the decision expire (zero to take the next measure immediately)
-* next_measure: which measure to trigger upon expiration of the current outcome; this must be either a string with the name of an **original** KYC measure, or a JSON object with the same information that is usually in ``[kyc-measures-]`` configuration sections (with an array of checks, context for each check, and another AML program).
+* combinator: "AND" if all of the 'next_measures' will eventually need to be satisfied, "OR" if those are choices and the user only has to satisfy one of them.
+* next_measures: measures to trigger upon expiration of the current outcome; array entries must be either a string with the name of an **original** context-free measure, or JSON objects with the same information that is usually in ``[kyc-measures-]`` configuration sections (with a check name, context, and AML program name).
 
 If the AML program fails (exits with a failure code or
 does not provide well-formed JSON output) the AML/KYC
@@ -548,19 +573,25 @@ Finally, the configuration specifies a set of
 
     [kyc-measure-$MEASURE_NAME]
 
-    # List of checks that could be run as part of
-    # this measure. The SPA should display *all*
-    # of these checks to the user, as they are given
-    # the choice.
-    # Each check is enabled by providing the
-    # context for the CHECK. The context can be
+    # Possible check for this measure.  Optional.
+    # If not given, PROGRAM should be run immediately
+    # (on an empty set of attributes).
+    CHECK_NAME = IB_FORM
+
+    # Context for the check.
+    # The context can be
     # just an empty JSON object if there is none.
-    CONTEXT_[$CHECK_NAME] = JSON
+    CONTEXT = {"choices":["individual","business"]}
 
     # Program to run on the context and check data to
     # determine the outcome and next measure.
     PROGRAM = taler-aml-program
 
+If no ``CHECK_NAME`` is provided at all, the
+AML ``PROGRAM`` is to be run immediately.  This is
+useful if no client-interaction is required to
+arrive at a decision.
+
 .. note::
 
    The list of *measures* is not complete: AML staff may freely define new
@@ -614,12 +645,14 @@ on GET ``/deposits/`` with the respective legitimization requirement row.
 
 .. sourcecode:: sql
 
-  CREATE TABLE IF NOT EXISTS wire_targets
-  (wire_target_serial_id BIGSERIAL UNIQUE
-  ,h_payto BYTEA NOT NULL CHECK (LENGTH(h_payto)=64),
-  ,payto_uri STRING NOT NULL
-  ,PRIMARY KEY (h_payto)
-  ) SHARD BY (h_payto);
+  CREATE TABLE wire_targets
+    (wire_target_serial_id BIGSERIAL UNIQUE
+    ,h_payto BYTEA NOT NULL CHECK (LENGTH(h_payto)=64),
+    ,payto_uri STRING NOT NULL
+    ,PRIMARY KEY (h_payto)
+    )
+    PARTITION BY HASH (h_payto);
+
   COMMENT ON TABLE wire_targets
     IS 'All recipients of money via the exchange';
   COMMENT ON COLUMN wire_targets.payto_uri
@@ -627,35 +660,167 @@ on GET ``/deposits/`` with the respective legitimization requirement row.
   COMMENT ON COLUMN wire_targets.h_payto
     IS 'Unsalted hash of payto_uri';
 
-  CREATE TABLE IF NOT EXISTS legitimization_requirements
-    (legitimization_requirement_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
+  CREATE TABLE IF NOT EXISTS legitimization_measures
+    (legitimization_measure_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
     ,h_payto BYTEA NOT NULL CHECK (LENGTH(h_payto)=32)
-    ,required_checks VARCHAR NOT NULL
-    ,UNIQUE (h_payto, required_checks);
-    ) PARTITION BY HASH (h_payto);
-
-  CREATE TABLE IF NOT EXISTS legitimization_processes
-    (legitimization_serial_id BIGSERIAL UNIQUE
+     REFERENCES wire_targets (h_payto)
+    ,start_time INT8 NOT NULL
+    ,jmeasures VARCHAR[] NOT NULL
+    ,is_and_combinator BOOL NOT NULL DEFAULT(FALSE)
+    ,is_finished BOOL NOT NULL DEFAULT(FALSE)
+    )
+    PARTITION BY HASH (h_payto);
+
+  COMMENT ON COLUMN legitimization_requirements.start_time
+    IS 'Time when the measure was triggered (by decision or rule)';
+  COMMENT ON COLUMN legitimization_requirements.is_finished
+    IS 'Set to TRUE if this set of measures was processed; used to avoid indexing measures that are done';
+  COMMENT ON COLUMN legitimization_requirements.is_and_combinator
+    IS 'Set to TRUE each of the measures will ultimately need to be satisfied; FALSE if the user has the choice to satisfy one of them';
+  COMMENT ON COLUMN legitimization_requirements.jmeasures
+    IS 'array with KYC/AML measures for the account encoded in JSON';
+
+  CREATE INDEX ON legitimization_measures (h_payto)
+    WHERE NOT finished;
+
+  CREATE TABLE legitimization_outcomes
+    (outcome_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
+    ,h_payto BYTEA CHECK (LENGTH(h_payto)=32)
+     REFERENCES wire_targets (h_payto)
+    ,decision_time INT8 NOT NULL DEFAULT(0)
+    ,expiration_time INT8 NOT NULL DEFAULT(0)
+    ,jproperties VARCHAR,
+    ,to_investigate BOOL NOT NULL
+    ,is_frozen BOOL NOT NULL
+    ,is_reported BOOL NOT NULL
+    ,is_active BOOL NOT NULL DEFAULT(TRUE)
+    ,new_rules NOT NULL TEXT
+    )
+    PARTITION BY HASH (h_payto);
+
+  COMMENT ON TABLE legitimization_outcomes
+    IS 'Outcomes can come from AML programs';
+  COMMENT ON COLUMN legitimization_outcomes.h_payto
+    IS 'hash of the payto://-URI this outcome is about';
+  COMMENT ON COLUMN legitimization_outcomes.decision_time
+    IS 'when was this outcome decided';
+  COMMENT ON COLUMN legitimization_outcomes.expiration_time
+    IS 'time when the decision expires and the expiration new_rules should be applied';
+  COMMENT ON COLUMN legitimization_outcomes.jproperties
+    IS 'JSON with account properties, such as PEP status, business domain, risk assessment, etc.';
+  COMMENT ON COLUMN legitimization_outcomes.to_investigate
+    IS 'AML staff should investigate the activity of this account';
+  COMMENT ON COLUMN legitimization_outcomes.is_frozen
+    IS 'Transactions with this account should be held (until expiration data or AML staff action)';
+  COMMENT ON COLUMN legitimization_outcomes.is_reported
+    IS 'Set to TRUE if the activity of the account was reported to authorities';
+  COMMENT ON COLUMN legitimization_outcomes.is_active
+    IS 'TRUE if this is the current authoritative legitimization outcome';
+  COMMENT ON COLUMN legitimization_outcomes.new_rules
+    IS 'JSON-encoding of KYC-rules to apply to the various operation types for this account; KYC check should first check if active new rules for a given account exist (and apply specified measures); if not, it should check the default rules to decide if a measure is required';
+
+  CREATE INDEX legitimization_outcomes_active
+    ON legitimization_outcomes(h_payto)
+    WHERE is_active;
+
+  CREATE TABLE kyc_setups
+    (kyc_setup_serial_id BIGSERIAL UNIQUE
     ,h_payto BYTEA NOT NULL CHECK (LENGTH(h_payto)=64)
+     REFERENCES wire_targets (h_payto)
+    ,start_time INT8 NOT NULL
     ,expiration_time INT8 NOT NULL DEFAULT (0)
+    ,legitimization_measure_serial_id BIGINT
+     REFERENCES legitimization_measures (legitimization_measure_serial_id)
+    ,measure_index INT8'
     ,provider_section VARCHAR NOT NULL
     ,provider_user_id VARCHAR DEFAULT NULL
     ,provider_legitimization_id VARCHAR DEFAULT NULL
-    ) PARTITION BY HASH (h_payto);
-
-  COMMENT ON COLUMN legitimizations.legitimization_serial_id
-    IS 'unique ID for this legitimization process at the exchange';
-  COMMENT ON COLUMN legitimizations.h_payto
-    IS 'foreign key linking the entry to the wire_targets table, NOT a primary key (multiple legitimizations are possible per wire target)';
-  COMMENT ON COLUMN legitimizations.expiration_time
-    IS 'in the future if the respective KYC check was passed successfully';
-  COMMENT ON COLUMN legitimizations.provider_section
+    ,redirect_url TEXT DEFAULT NULL
+    ,finished BOOLEAN DEFAULT (FALSE)
+    )
+    PARTITION BY HASH (h_payto);
+
+  COMMENT ON TABLE kyc_setups
+    IS 'here we track KYC processes we initiated with external providers; the main reason is so that we do not initiate a second process when an equivalent one is still active; note that h_payto, provider_section, jcontext must match and the process must not be finished or expired for an existing redirect_url to be re-used; given that clients may voluntarily initiate KYC processes, there may not always be a legitimization_measure that triggered the setup';
+  COMMENT ON COLUMN kyc_setups.h_payto
+    IS 'foreign key linking the entry to the wire_targets table, NOT a primary key (multiple KYC setups are possible per wire target)';
+  COMMENT ON COLUMN kyc_setups.start_time
+    IS 'when was the legitimization process initiated';
+  COMMENT ON COLUMN kyc_setups.expiration_time
+    IS 'when does the process expire (and needs to be manually set up again)';
+  COMMENT ON COLUMN kyc_setups.measure_index
+    IS 'index of the measure in legitimization_measures that was selected for this KYC setup; NULL if legitimization_measure_serial_id is NULL; enables determination of the context data provided to the external process';
+  COMMENT ON COLUMN kyc_setups.provider_section
     IS 'Configuration file section with details about this provider';
-  COMMENT ON COLUMN legitimizations.provider_user_id
+  COMMENT ON COLUMN kyc_setups.provider_user_id
     IS 'Identifier for the user at the provider that was used for the legitimization. NULL if provider is unaware.';
-  COMMENT ON COLUMN legitimizations.provider_legitimization_id
+  COMMENT ON COLUMN kyc_setups.provider_legitimization_id
     IS 'Identifier for the specific legitimization process at the provider. NULL if legitimization was not started.';
+  COMMENT ON COLUMN kyc_setups.legitimization_measure_serial_id
+    IS 'measure that enabled this setup, NULL if client voluntarily initiated the process';
+  COMMENT ON COLUMN kyc_setups.redirect_url
+    IS 'Where the user should be redirected for this external KYC process';
+  COMMENT ON COLUMN kyc_setups.finished
+    IS 'set to TRUE when the specific legitimization process is finished';
+
+  CREATE TABLE kyc_attributes
+    (kyc_attributes_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
+    ,h_payto BYTEA PRIMARY KEY CHECK (LENGTH(h_payto)=32)
+     REFERENCES wire_targets (h_payto)
+    ,kyc_prox BYTEA NOT NULL CHECK (LENGTH(kyc_prox)=32)
+    ,kyc_setup_serial_id INT8 NOT NULL
+     REFERENCES kyc_setups (kyc_setup_serial_id)
+    ,collection_time INT8 NOT NULL
+    ,expiration_time INT8 NOT NULL
+    ,trigger_outcome_serial INT8 NOT NULL
+     REFERENCES legitimization_outcomes(outcome_serial_id)
+    ,encrypted_attributes BYTEA NOT NULL
+    ) PARTITION BY HASH (h_payto);
 
+  COMMENT ON COLUMN kyc_attributes.h_payto
+    IS 'identifies the account this is about';
+  COMMENT ON COLUMN kyc_attributes.kyc_prox
+    IS 'for proximity search on encrypted data';
+  COMMENT ON COLUMN kyc_attributes.kyc_setup_serial_id
+    IS 'serial ID of the KYC setup that resulted in these attributes';
+  COMMENT ON COLUMN kyc_attributes.collection_time
+    IS 'when were these attributes collected';
+  COMMENT ON COLUMN kyc_attributes.expiration_time
+    IS 'when are these attributes expected to expire';
+  COMMENT ON COLUMN kyc_attributes.trigger_outcome_serial
+    IS 'ID of the outcome that was returned by the AML program based on the KYC data collected';
+  COMMENT ON COLUMN kyc_attributes.encrypted_attributes
+    IS 'encrypted JSON object with the attribute data the check provided';
+
+  CREATE TABLE aml_history
+    (aml_history_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
+    ,h_payto BYTEA CHECK (LENGTH(h_payto)=32)
+     REFERENCES wire_targets (h_payto)
+    ,legitimization_outcome INT8 NOT NULL
+     REFERENCES legitimization_outcomes (outcome_serial_id)
+    ,justification TEXT NOT NULL
+    ,decider_pub BYTEA CHECK (LENGTH(decider_pub)=32)
+    ,decider_sig BYTEA CHECK (LENGTH(decider_sig)=64);
+
+  COMMENT ON TABLE aml_history
+    IS 'Records decisions by AML staff with the respective signature and free-form justification.';
+  COMMENT ON COLUMN aml_history.legitimization_outcome
+    IS 'Actual outcome for the account (included in what decider_sig signs over)';
+  COMMENT ON COLUMN aml_history.decider_sig
+    IS 'Signature key of the staff member affirming the AML decision; of type AML_DECISION';
+
+  CREATE TABLE kyc_events
+    (event_serial_id BIGINT GENERATED BY DEFAULT AS IDENTITY
+    ,event_timestamp INT8 NOT NULL
+    ,event_type TEXT NOT NULL);
+
+  COMMENT ON TABLE kyc_events
+    IS 'Records of key events for statistics. Populated via triggers.';
+  COMMENT ON COLUMN kyc_events.event_type
+    IS 'Name of the event, such as account-open or sar-filed';
+
+  CREATE INDEX kyc_event_index
+    ON kyc_events(event_type,event_timestamp);
 
 KYC forms
 ^^^^^^^^^