commit aa97d3af2443e20190dc163959816cc49d9c837b
parent ad2405b87647754aed6a26532ca457fcaea0ed2a
Author: Christian Grothoff <christian@grothoff.org>
Date: Mon, 12 Oct 2026 22:48:18 +0200
fix handling out-of-order row-ids in sanction list checker
Diffstat:
20 files changed, 954 insertions(+), 475 deletions(-)
diff --git a/src/auditor/taler-auditor-sync.c b/src/auditor/taler-auditor-sync.c
@@ -157,7 +157,6 @@ struct InsertContext;
#include "exchange-database/iterate_aggregation_amounts_for_kyc_check.h"
#include "exchange-database/iterate_aggregations_above_serial_id.h"
#include "exchange-database/get_aggregation_transient.h"
-#include "exchange-database/iterate_all_kyc_attributes.h"
#include "exchange-database/iterate_all_purse_decisions_above_serial_id.h"
#include "exchange-database/iterate_all_purse_deletions_above_serial_id.h"
#include "exchange-database/iterate_aml_attributes.h"
diff --git a/src/exchange/exchange.conf b/src/exchange/exchange.conf
@@ -148,8 +148,11 @@ AML_SPA_DIR = $PREFIX/share/taler-exchange-aml-webui/
KYC_SPA_DIR = $PREFIX/share/taler-exchange-kyc-webui/
[exchange-sanctionscheck]
-# Name where we store the sanctions check offset.
-MIN_ROW_FILENAME = ${HOME}/.cache/sanctionscheck-offset.bin
+# Name of the file where we store the hash of the sanctions list
+# (RATER_COMMAND and the contents of the files it names) that the
+# KYC attributes were screened against. If the hash changes, all
+# KYC attributes are screened again.
+LIST_HASH_FILENAME = ${HOME}/.cache/sanctionscheck-list-hash.bin
# Sanction list match rating that must be exceeded for an automated
# freeze of the account (without manual investigation first).
diff --git a/src/exchange/taler-exchange-sanctionscheck.c b/src/exchange/taler-exchange-sanctionscheck.c
@@ -1,6 +1,6 @@
/*
This file is part of TALER
- Copyright (C) 2025 Taler Systems SA
+ Copyright (C) 2025, 2026 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU Affero General Public License as published by the Free Software
@@ -34,7 +34,9 @@
#include "exchange-database/insert_sanction_list_hit.h"
#include "exchange-database/start.h"
#include "exchange-database/rollback.h"
-#include "exchange-database/iterate_all_kyc_attributes.h"
+#include "exchange-database/iterate_unscreened_kyc_attributes.h"
+#include "exchange-database/update_to_kyc_attributes_screened.h"
+#include "exchange-database/update_to_kyc_attributes_unscreened.h"
#include "exchange-database/event_listen.h"
#include "exchange-database/event_listen_cancel.h"
#include "exchange-database/start.h"
@@ -120,23 +122,10 @@ static struct Account *acc_head;
static struct Account *acc_tail;
/**
- * Minimum row ID to process records from. This is the value we persist,
- * and it is only advanced once the transaction that evaluated the
- * respective rows was committed.
+ * Set to true if marking a row as screened failed while iterating over
+ * the rows of the current transaction.
*/
-static uint64_t min_row_id;
-
-/**
- * Highest row ID evaluated in the current transaction. Folded into
- * #min_row_id when that transaction commits.
- */
-static uint64_t max_row_id;
-
-/**
- * File descriptor with the name of the file where we track our
- * progress.
- */
-static int min_row_fd = -1;
+static bool mark_failed;
/**
* '-t' command line flag. Disables background processing mode.
@@ -144,13 +133,13 @@ static int min_row_fd = -1;
static int testmode;
/**
- * '-r' command line flag. Restarts analysis from scratch (for
- * fresh sanction list).
+ * '-r' command line flag. Restarts analysis from scratch even if
+ * the sanctions list seems unchanged.
*/
static int reset;
/**
- * '-n' command line flag. Do not actually run, only reset.
+ * '-n' command line flag. Do not actually run, only reset if needed.
*/
static int norun;
@@ -199,33 +188,6 @@ static float investigation_limit = 0.9;
static float investigation_confidence_limit = 0.0;
/**
- * Write @a min_row_id to @a min_row_fd.
- */
-static void
-sync_row (void)
-{
- uint64_t r = GNUNET_htonll (min_row_id);
- ssize_t rval;
-
- GNUNET_assert (-1 != min_row_fd);
- GNUNET_break (0 == lseek (min_row_fd,
- 0,
- SEEK_SET));
- rval = write (min_row_fd,
- &r,
- sizeof (r));
- if (rval != (ssize_t) sizeof (r))
- {
- GNUNET_break (-1 == rval);
- GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING,
- "write");
- }
- GNUNET_break (0 ==
- fsync (min_row_fd));
-}
-
-
-/**
* We're being aborted with CTRL-C (or SIGTERM). Shut down.
*
* @param cls closure
@@ -236,12 +198,6 @@ shutdown_task (void *cls)
struct Account *acc;
(void) cls;
- if (-1 != min_row_fd)
- {
- sync_row ();
- GNUNET_break (0 == close (min_row_fd));
- min_row_fd = -1;
- }
while (NULL != (acc = acc_head))
{
GNUNET_CONTAINER_DLL_remove (acc_head,
@@ -299,6 +255,65 @@ begin_transaction (void);
/**
+ * Mark the attributes in @a row_id as screened against the current
+ * sanctions list, as part of the current transaction.
+ *
+ * @param h_payto account the attributes are about
+ * @param row_id row of the attributes in the kyc_attributes table
+ * @return true on success
+ */
+static bool
+mark_screened (const struct TALER_NormalizedPaytoHashP *h_payto,
+ uint64_t row_id)
+{
+ enum GNUNET_DB_QueryStatus qs;
+
+ qs = TALER_EXCHANGEDB_update_to_kyc_attributes_screened (pg,
+ h_payto,
+ row_id);
+ if (qs < 0)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Failed to mark KYC attributes #%llu as screened\n",
+ (unsigned long long) row_id);
+ return false;
+ }
+ /* If the row is gone (garbage collected), there is nothing to mark. */
+ return true;
+}
+
+
+/**
+ * All rows of the current transaction have been screened: commit it,
+ * and immediately start the next one if new KYC attributes were
+ * stored in the meantime.
+ */
+static void
+commit_transaction (void)
+{
+ enum GNUNET_DB_QueryStatus qs;
+
+ qs = TALER_EXCHANGEDB_commit (pg);
+ if (qs < 0)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Failed to commit DB transaction\n");
+ global_ret = EXIT_NOTCONFIGURED;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ in_transaction = false;
+ if (restart_now)
+ {
+ begin_transaction ();
+ return;
+ }
+ if (testmode)
+ GNUNET_SCHEDULER_shutdown ();
+}
+
+
+/**
* Function called with the result of a sanction evaluation.
*
* @param cls closure
@@ -380,13 +395,15 @@ sanction_cb (void *cls,
GNUNET_assert (0 ==
json_object_update_missing (properties,
acc->properties));
- qs = TALER_EXCHANGEDB_insert_sanction_list_hit (pg,
- &acc->h_payto,
- investigate,
- new_rules,
- properties,
- 1,
- events);
+ qs = TALER_EXCHANGEDB_insert_sanction_list_hit (
+ pg,
+ &acc->h_payto,
+ acc->row_id,
+ investigate,
+ new_rules,
+ properties,
+ 1,
+ events);
json_decref (properties);
if (qs < 0)
{
@@ -402,8 +419,6 @@ sanction_cb (void *cls,
acc);
json_decref (acc->properties);
/* Evaluations complete out of order, so keep the highest row we did. */
- max_row_id = GNUNET_MAX (max_row_id,
- acc->row_id);
GNUNET_free (acc);
if (NULL != acc_head)
return; /* more work */
@@ -423,9 +438,6 @@ sanction_cb (void *cls,
}
}
in_transaction = false;
- /* Our writes are durable now, so the resume point may move. */
- min_row_id = max_row_id;
- sync_row ();
if (restart_now)
{
begin_transaction ();
@@ -471,6 +483,13 @@ account_cb (void *cls,
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Skipping %llu as suppressed by staff as false-positive\n",
(unsigned long long) row_id);
+ /* Done with this row for the current sanctions list. */
+ if (! mark_screened (h_payto,
+ row_id))
+ {
+ mark_failed = true;
+ return false;
+ }
return true;
}
attributes = TALER_CRYPTO_kyc_attributes_decrypt (&attribute_key,
@@ -481,6 +500,14 @@ account_cb (void *cls,
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Failed to decrypt attributes at row #%llu\n",
(unsigned long long) row_id);
+ /* Do not try again (and again log an error) on every round; the row
+ is screened again with the next sanctions list or with -r. */
+ if (! mark_screened (h_payto,
+ row_id))
+ {
+ mark_failed = true;
+ return false;
+ }
return true;
}
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
@@ -588,7 +615,7 @@ begin_transaction ()
enum GNUNET_DB_QueryStatus qs;
restart_now = false;
- max_row_id = min_row_id;
+ mark_failed = false;
GNUNET_assert (! in_transaction);
if (GNUNET_OK !=
TALER_EXCHANGEDB_start (pg,
@@ -603,10 +630,9 @@ begin_transaction ()
in_transaction = true;
/* FIXME-10063: we may want to eventually limit the number of
records we process in a single transaction. */
- qs = TALER_EXCHANGEDB_iterate_all_kyc_attributes (pg,
- min_row_id,
- &account_cb,
- NULL);
+ qs = TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (pg,
+ &account_cb,
+ NULL);
if (qs < 0)
{
global_ret = EXIT_FAILURE;
@@ -614,21 +640,239 @@ begin_transaction ()
GNUNET_SCHEDULER_shutdown ();
return;
}
+ if (mark_failed)
+ {
+ global_ret = EXIT_FAILURE;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
{
TALER_EXCHANGEDB_rollback (pg);
+ in_transaction = false;
if (testmode)
GNUNET_SCHEDULER_shutdown ();
return;
}
if (NULL == acc_head)
- in_transaction = false;
- if ( (NULL == acc_head) &&
- (testmode) )
{
- /* no work, not incremental, we are done */
- GNUNET_SCHEDULER_shutdown ();
+ /* Every row was skipped (and marked as screened), so no evaluation
+ will call back to commit: do it here. */
+ commit_transaction ();
+ }
+}
+
+
+/**
+ * Compute a hash over the sanctions list the rater is going to use. The
+ * rater loads the list itself, usually from a file named on its command
+ * line ("robocop $LIST.json"), so we hash the command line @a sargv and
+ * the contents of every file named as one of its arguments.
+ *
+ * @param sargc number of entries in @a sargv
+ * @param sargv command line of the rater
+ * @param[out] list_hash set to the hash
+ * @return #GNUNET_OK on success
+ */
+static enum GNUNET_GenericReturnValue
+hash_sanction_list (unsigned int sargc,
+ char *const *sargv,
+ struct GNUNET_HashCode *list_hash)
+{
+ struct GNUNET_HashContext *hc;
+
+ hc = GNUNET_CRYPTO_hash_context_start ();
+ for (unsigned int i = 0; i<sargc; i++)
+ {
+ struct GNUNET_DISK_FileHandle *fh;
+ char buf[65536];
+ ssize_t ret;
+
+ GNUNET_CRYPTO_hash_context_read (hc,
+ sargv[i],
+ strlen (sargv[i]) + 1);
+ /* sargv[0] is the rater program, not part of the list */
+ if ( (0 == i) ||
+ (GNUNET_YES !=
+ GNUNET_DISK_file_test (sargv[i])) )
+ continue;
+ fh = GNUNET_DISK_file_open (sargv[i],
+ GNUNET_DISK_OPEN_READ,
+ GNUNET_DISK_PERM_NONE);
+ if (NULL == fh)
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "open",
+ sargv[i]);
+ GNUNET_CRYPTO_hash_context_abort (hc);
+ return GNUNET_SYSERR;
+ }
+ while (0 < (ret = GNUNET_DISK_file_read (fh,
+ buf,
+ sizeof (buf))))
+ GNUNET_CRYPTO_hash_context_read (hc,
+ buf,
+ (size_t) ret);
+ GNUNET_break (GNUNET_OK ==
+ GNUNET_DISK_file_close (fh));
+ if (0 > ret)
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "read",
+ sargv[i]);
+ GNUNET_CRYPTO_hash_context_abort (hc);
+ return GNUNET_SYSERR;
+ }
+ }
+ GNUNET_CRYPTO_hash_context_finish (hc,
+ list_hash);
+ return GNUNET_OK;
+}
+
+
+/**
+ * Mark all KYC attributes as not yet screened, so that all of them are
+ * screened against the new sanctions list.
+ *
+ * @return #GNUNET_OK on success
+ */
+static enum GNUNET_GenericReturnValue
+unscreen_all (void)
+{
+ enum GNUNET_DB_QueryStatus qs;
+
+ if (GNUNET_OK !=
+ TALER_EXCHANGEDB_start (pg,
+ "sanctionscheck-unscreen"))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Failed to begin DB transaction\n");
+ return GNUNET_SYSERR;
+ }
+ qs = TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened (pg);
+ if (qs < 0)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Failed to mark KYC attributes as unscreened\n");
+ TALER_EXCHANGEDB_rollback (pg);
+ return GNUNET_SYSERR;
+ }
+ if (0 > TALER_EXCHANGEDB_commit (pg))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Failed to commit DB transaction\n");
+ return GNUNET_SYSERR;
+ }
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Marked %llu screened KYC attribute rows for screening again\n",
+ (unsigned long long) qs);
+ return GNUNET_OK;
+}
+
+
+/**
+ * Check if the sanctions list changed since the KYC attributes were
+ * screened (or if the operator asked for a reset with -r), and if so,
+ * mark all of them as unscreened. Remembers @a list_hash in the file
+ * named by the LIST_HASH_FILENAME option as the list that the "screened"
+ * markers in the database now refer to. Sets #global_ret on failure.
+ *
+ * @param list_hash hash over the current sanctions list
+ * @return #GNUNET_OK on success
+ */
+static enum GNUNET_GenericReturnValue
+check_new_sanction_list (const struct GNUNET_HashCode *list_hash)
+{
+ struct GNUNET_HashCode old_hash;
+ char *fn;
+ int fd;
+
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_filename (cfg,
+ "exchange-sanctionscheck",
+ "LIST_HASH_FILENAME",
+ &fn))
+ {
+ GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+ "exchange-sanctionscheck",
+ "LIST_HASH_FILENAME");
+ global_ret = EXIT_NOTCONFIGURED;
+ return GNUNET_SYSERR;
+ }
+ if (GNUNET_OK !=
+ GNUNET_DISK_directory_create_for_file (fn))
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "mkdir",
+ fn);
+ GNUNET_free (fn);
+ global_ret = EXIT_NOPERMISSION;
+ return GNUNET_SYSERR;
+ }
+ /* Open (and thus check that we can write) the file before touching
+ the database. */
+ fd = open (fn,
+ O_CREAT | O_RDWR,
+ S_IRUSR | S_IWUSR);
+ if (-1 == fd)
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "open",
+ fn);
+ GNUNET_free (fn);
+ global_ret = EXIT_NOTCONFIGURED;
+ return GNUNET_SYSERR;
+ }
+ if ( (! reset) &&
+ (sizeof (old_hash) ==
+ read (fd,
+ &old_hash,
+ sizeof (old_hash))) &&
+ (0 == GNUNET_memcmp (&old_hash,
+ list_hash)) )
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Sanctions list unchanged, screening new KYC attributes\n");
+ GNUNET_break (0 == close (fd));
+ GNUNET_free (fn);
+ return GNUNET_OK;
+ }
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ reset
+ ? "Reset requested, screening all KYC attributes again\n"
+ : "New sanctions list, screening all KYC attributes again\n");
+ /* Database first: if we fail after this, the old hash in the file
+ only makes us reset (needlessly) once more on the next start. */
+ if (GNUNET_OK !=
+ unscreen_all ())
+ {
+ GNUNET_break (0 == close (fd));
+ GNUNET_free (fn);
+ global_ret = EXIT_FAILURE;
+ return GNUNET_SYSERR;
+ }
+ if ( (0 != lseek (fd,
+ 0,
+ SEEK_SET)) ||
+ (sizeof (*list_hash) !=
+ write (fd,
+ list_hash,
+ sizeof (*list_hash))) ||
+ (0 != ftruncate (fd,
+ sizeof (*list_hash))) ||
+ (0 != fsync (fd)) )
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "write",
+ fn);
+ GNUNET_break (0 == close (fd));
+ GNUNET_free (fn);
+ global_ret = EXIT_FAILURE;
+ return GNUNET_SYSERR;
}
+ GNUNET_break (0 == close (fd));
+ GNUNET_free (fn);
+ return GNUNET_OK;
}
@@ -668,6 +912,8 @@ run (void *cls,
const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *c)
{
+ struct GNUNET_HashCode list_hash;
+
(void) cls;
(void) cfgfile;
cfg = c;
@@ -794,6 +1040,22 @@ run (void *cls,
GNUNET_SCHEDULER_shutdown ();
return;
}
+ /* Before starting the rater, so a list that changes while the rater
+ loads it only results in one (needless) reset too many. */
+ if (GNUNET_OK !=
+ hash_sanction_list (sargc,
+ sargv,
+ &list_hash))
+ {
+ for (unsigned int i = 0; i<sargc; i++)
+ GNUNET_free (sargv[i]);
+ GNUNET_array_grow (sargv,
+ sargc,
+ 0);
+ global_ret = EXIT_NOPERMISSION;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
GNUNET_array_append (sargv,
sargc,
NULL);
@@ -811,74 +1073,11 @@ run (void *cls,
return;
}
}
+ if (GNUNET_OK !=
+ check_new_sanction_list (&list_hash))
{
- char *min_row_fn;
- uint64_t r;
-
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_filename (cfg,
- "exchange-sanctionscheck",
- "MIN_ROW_FILENAME",
- &min_row_fn))
- {
- GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
- "exchange-sanctionscheck",
- "MIN_ROW_FILENAME");
- global_ret = EXIT_NOTCONFIGURED;
- GNUNET_SCHEDULER_shutdown ();
- return;
- }
- if (reset &&
- (0 != unlink (min_row_fn)) &&
- (ENOENT != errno) )
- {
- GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
- "unlink",
- min_row_fn);
- GNUNET_free (min_row_fn);
- global_ret = EXIT_NOPERMISSION;
- GNUNET_SCHEDULER_shutdown ();
- return;
- }
- if (GNUNET_OK !=
- GNUNET_DISK_directory_create_for_file (min_row_fn))
- {
- GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
- "open",
- min_row_fn);
- GNUNET_free (min_row_fn);
- global_ret = EXIT_NOPERMISSION;
- GNUNET_SCHEDULER_shutdown ();
- return;
- }
- min_row_fd = open (min_row_fn,
- O_CREAT | O_RDWR,
- S_IRUSR | S_IWUSR);
- if (-1 == min_row_fd)
- {
- GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
- "open",
- min_row_fn);
- GNUNET_free (min_row_fn);
- global_ret = EXIT_NOTCONFIGURED;
- GNUNET_SCHEDULER_shutdown ();
- return;
- }
- if (sizeof (r) !=
- read (min_row_fd,
- &r,
- sizeof (r)))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Could not read starting row from `%s', will start from 0\n",
- min_row_fn);
- min_row_id = 0;
- }
- else
- {
- min_row_id = GNUNET_ntohll (r);
- }
- GNUNET_free (min_row_fn);
+ GNUNET_SCHEDULER_shutdown ();
+ return;
}
if (norun)
{
@@ -922,7 +1121,7 @@ main (int argc,
&norun),
GNUNET_GETOPT_option_flag ('r',
"reset",
- "rescan all records (to be used when the sanction list was updated)",
+ "rescan all records, even if the sanctions list seems unchanged",
&reset),
GNUNET_GETOPT_option_flag ('t',
"test",
diff --git a/src/exchangedb/insert_sanction_list_hit.c b/src/exchangedb/insert_sanction_list_hit.c
@@ -27,6 +27,7 @@ enum GNUNET_DB_QueryStatus
TALER_EXCHANGEDB_insert_sanction_list_hit (
struct TALER_EXCHANGEDB_PostgresContext *pg,
const struct TALER_NormalizedPaytoHashP *h_payto,
+ uint64_t kyc_attributes_serial_id,
bool to_investigate,
const json_t *new_rules,
const json_t *account_properties,
@@ -48,6 +49,7 @@ TALER_EXCHANGEDB_insert_sanction_list_hit (
GNUNET_PQ_query_param_auto_from_type (h_payto),
GNUNET_PQ_query_param_timestamp (&now),
GNUNET_PQ_query_param_timestamp (&never),
+ GNUNET_PQ_query_param_uint64 (&kyc_attributes_serial_id),
NULL != account_properties
? TALER_PQ_query_param_json (account_properties)
: GNUNET_PQ_query_param_null (),
@@ -74,8 +76,8 @@ TALER_EXCHANGEDB_insert_sanction_list_hit (
"SELECT"
" out_outcome_serial_id AS outcome_serial_id"
" FROM exchange_do_insert_sanction_list_hit"
- "($1,$2,$3,$4::TEXT::JSONB,$5::TEXT::JSONB,$6,$7,$8);");
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "($1,$2,$3,$4,$5::TEXT::JSONB,$6::TEXT::JSONB,$7,$8,$9);");
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Inserting LEGI OUTCOME from sanction list hit\n");
qs = GNUNET_PQ_eval_prepared_singleton_select (
pg->conn,
diff --git a/src/exchangedb/insert_sanction_list_hit.sql b/src/exchangedb/insert_sanction_list_hit.sql
@@ -19,6 +19,7 @@ CREATE FUNCTION exchange_do_insert_sanction_list_hit(
IN in_h_normalized_payto BYTEA,
IN in_decision_time INT8,
IN in_expiration_time INT8,
+ IN in_kyc_attributes_serial_id INT8,
IN in_properties JSONB, -- can be NULL
IN in_new_rules JSONB, -- can be NULL
IN in_to_investigate BOOLEAN,
@@ -63,6 +64,12 @@ THEN
END IF;
END IF;
+-- Mark sanction list entry as screened
+UPDATE kyc_attributes
+ SET screened=TRUE
+ WHERE h_payto=in_h_normalized_payto
+ AND kyc_attributes_serial_id=in_kyc_attributes_serial_id;
+
-- Disable all previous legitimization outcomes.
UPDATE legitimization_outcomes
SET is_active=FALSE
@@ -109,5 +116,5 @@ EXECUTE FORMAT (
END $$;
-COMMENT ON FUNCTION exchange_do_insert_sanction_list_hit(BYTEA, INT8, INT8, JSONB, JSONB, BOOLEAN, TEXT, TEXT[])
+COMMENT ON FUNCTION exchange_do_insert_sanction_list_hit(BYTEA, INT8, INT8, INT8, JSONB, JSONB, BOOLEAN, TEXT, TEXT[])
IS 'Insert result from sanction list check into the table';
diff --git a/src/exchangedb/iterate_all_kyc_attributes.c b/src/exchangedb/iterate_all_kyc_attributes.c
@@ -1,174 +0,0 @@
-/*
- This file is part of TALER
- Copyright (C) 2025 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU General Public License as published by the Free Software
- Foundation; either version 3, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
- */
-/**
- * @file exchangedb/iterate_all_kyc_attributes.c
- * @brief Implementation of the iterate_all_kyc_attributes function for Postgres
- * @author Christian Grothoff
- */
-#include "taler/taler_pq_lib.h"
-#include "exchange-database/iterate_all_kyc_attributes.h"
-#include "helper.h"
-
-/**
- * Closure for #get_all_attributes_cb().
- */
-struct GetAttributesContext
-{
- /**
- * Function to call per result.
- */
- TALER_EXCHANGEDB_AllAttributesCallback cb;
-
- /**
- * Closure for @e cb.
- */
- void *cb_cls;
-
- /**
- * Plugin context.
- */
- struct TALER_EXCHANGEDB_PostgresContext *pg;
-
- /**
- * Flag set to #GNUNET_OK as long as everything is fine.
- */
- enum GNUNET_GenericReturnValue status;
-
-};
-
-/**
- * Invoke the callback for each result.
- *
- * @param cls a `struct GetAttributesContext *`
- * @param result SQL result
- * @param num_results number of rows in @a result
- */
-static void
-get_attributes_cb (void *cls,
- PGresult *result,
- unsigned int num_results)
-{
- struct GetAttributesContext *ctx = cls;
-
- for (unsigned int i = 0; i < num_results; i++)
- {
- uint64_t rowid;
- struct GNUNET_TIME_Timestamp collection_time;
- struct GNUNET_TIME_Timestamp expiration_time;
- struct TALER_NormalizedPaytoHashP h_payto;
- json_t *properties = NULL;
- size_t enc_attributes_size;
- void *enc_attributes;
- char *provider;
- struct GNUNET_PQ_ResultSpec rs[] = {
- GNUNET_PQ_result_spec_string ("provider_name",
- &provider),
- GNUNET_PQ_result_spec_uint64 ("kyc_attributes_serial_id",
- &rowid),
- GNUNET_PQ_result_spec_allow_null (
- TALER_PQ_result_spec_json ("jproperties",
- &properties),
- NULL),
- GNUNET_PQ_result_spec_timestamp ("collection_time",
- &collection_time),
- GNUNET_PQ_result_spec_timestamp ("expiration_time",
- &expiration_time),
- GNUNET_PQ_result_spec_auto_from_type ("h_payto",
- &h_payto),
- GNUNET_PQ_result_spec_variable_size ("encrypted_attributes",
- &enc_attributes,
- &enc_attributes_size),
- GNUNET_PQ_result_spec_end
- };
-
- if (GNUNET_OK !=
- GNUNET_PQ_extract_result (result,
- rs,
- i))
- {
- GNUNET_break (0);
- ctx->status = GNUNET_SYSERR;
- return;
- }
- {
- bool cont;
-
- cont = ctx->cb (ctx->cb_cls,
- rowid,
- &h_payto,
- provider,
- collection_time,
- expiration_time,
- properties,
- enc_attributes_size,
- enc_attributes);
- GNUNET_PQ_cleanup_result (rs);
- if (! cont)
- return;
- }
- }
-}
-
-
-enum GNUNET_DB_QueryStatus
-TALER_EXCHANGEDB_iterate_all_kyc_attributes (
- struct TALER_EXCHANGEDB_PostgresContext *pg,
- uint64_t min_row_id,
- TALER_EXCHANGEDB_AllAttributesCallback cb,
- void *cb_cls)
-{
- struct GNUNET_PQ_QueryParam params[] = {
- GNUNET_PQ_query_param_uint64 (&min_row_id),
- GNUNET_PQ_query_param_end
- };
- struct GetAttributesContext ctx = {
- .cb = cb,
- .cb_cls = cb_cls,
- .pg = pg,
- .status = GNUNET_OK
- };
- enum GNUNET_DB_QueryStatus qs;
-
- PREPARE (pg,
- "iterate_all_kyc_attributes",
- "SELECT "
- " lp.provider_name"
- ",ka.h_payto"
- ",ka.kyc_attributes_serial_id"
- ",lo.jproperties::TEXT"
- ",ka.collection_time"
- ",ka.expiration_time"
- ",ka.encrypted_attributes"
- " FROM kyc_attributes ka"
- " JOIN legitimization_processes lp"
- " ON (ka.legitimization_serial = lp.legitimization_process_serial_id)"
- " LEFT JOIN legitimization_outcomes lo"
- " ON (ka.h_payto = lo.h_payto)"
- /* **IF** we joined with 'lo', the lo must be active */
- " WHERE COALESCE(lo.is_active,TRUE)"
- " AND kyc_attributes_serial_id > $1"
- " ORDER BY kyc_attributes_serial_id ASC"
- );
- qs = GNUNET_PQ_eval_prepared_multi_select (
- pg->conn,
- "iterate_all_kyc_attributes",
- params,
- &get_attributes_cb,
- &ctx);
- if (GNUNET_OK != ctx.status)
- return GNUNET_DB_STATUS_HARD_ERROR;
- return qs;
-}
diff --git a/src/exchangedb/iterate_unscreened_kyc_attributes.c b/src/exchangedb/iterate_unscreened_kyc_attributes.c
@@ -0,0 +1,172 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2025 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+/**
+ * @file exchangedb/iterate_unscreened_kyc_attributes.c
+ * @brief Implementation of the iterate_unscreened_kyc_attributes function for Postgres
+ * @author Christian Grothoff
+ */
+#include "taler/taler_pq_lib.h"
+#include "exchange-database/iterate_unscreened_kyc_attributes.h"
+#include "helper.h"
+
+/**
+ * Closure for #get_all_attributes_cb().
+ */
+struct GetAttributesContext
+{
+ /**
+ * Function to call per result.
+ */
+ TALER_EXCHANGEDB_UnscreenedAttributesCallback cb;
+
+ /**
+ * Closure for @e cb.
+ */
+ void *cb_cls;
+
+ /**
+ * Plugin context.
+ */
+ struct TALER_EXCHANGEDB_PostgresContext *pg;
+
+ /**
+ * Flag set to #GNUNET_OK as long as everything is fine.
+ */
+ enum GNUNET_GenericReturnValue status;
+
+};
+
+/**
+ * Invoke the callback for each result.
+ *
+ * @param cls a `struct GetAttributesContext *`
+ * @param result SQL result
+ * @param num_results number of rows in @a result
+ */
+static void
+get_attributes_cb (void *cls,
+ PGresult *result,
+ unsigned int num_results)
+{
+ struct GetAttributesContext *ctx = cls;
+
+ for (unsigned int i = 0; i < num_results; i++)
+ {
+ uint64_t rowid;
+ struct GNUNET_TIME_Timestamp collection_time;
+ struct GNUNET_TIME_Timestamp expiration_time;
+ struct TALER_NormalizedPaytoHashP h_payto;
+ json_t *properties = NULL;
+ size_t enc_attributes_size;
+ void *enc_attributes;
+ char *provider;
+ struct GNUNET_PQ_ResultSpec rs[] = {
+ GNUNET_PQ_result_spec_string ("provider_name",
+ &provider),
+ GNUNET_PQ_result_spec_uint64 ("kyc_attributes_serial_id",
+ &rowid),
+ GNUNET_PQ_result_spec_allow_null (
+ TALER_PQ_result_spec_json ("jproperties",
+ &properties),
+ NULL),
+ GNUNET_PQ_result_spec_timestamp ("collection_time",
+ &collection_time),
+ GNUNET_PQ_result_spec_timestamp ("expiration_time",
+ &expiration_time),
+ GNUNET_PQ_result_spec_auto_from_type ("h_payto",
+ &h_payto),
+ GNUNET_PQ_result_spec_variable_size ("encrypted_attributes",
+ &enc_attributes,
+ &enc_attributes_size),
+ GNUNET_PQ_result_spec_end
+ };
+
+ if (GNUNET_OK !=
+ GNUNET_PQ_extract_result (result,
+ rs,
+ i))
+ {
+ GNUNET_break (0);
+ ctx->status = GNUNET_SYSERR;
+ return;
+ }
+ {
+ bool cont;
+
+ cont = ctx->cb (ctx->cb_cls,
+ rowid,
+ &h_payto,
+ provider,
+ collection_time,
+ expiration_time,
+ properties,
+ enc_attributes_size,
+ enc_attributes);
+ GNUNET_PQ_cleanup_result (rs);
+ if (! cont)
+ return;
+ }
+ }
+}
+
+
+enum GNUNET_DB_QueryStatus
+TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ struct TALER_EXCHANGEDB_PostgresContext *pg,
+ TALER_EXCHANGEDB_UnscreenedAttributesCallback cb,
+ void *cb_cls)
+{
+ struct GNUNET_PQ_QueryParam params[] = {
+ GNUNET_PQ_query_param_end
+ };
+ struct GetAttributesContext ctx = {
+ .cb = cb,
+ .cb_cls = cb_cls,
+ .pg = pg,
+ .status = GNUNET_OK
+ };
+ enum GNUNET_DB_QueryStatus qs;
+
+ PREPARE (pg,
+ "iterate_unscreened_kyc_attributes",
+ "SELECT "
+ " lp.provider_name"
+ ",ka.h_payto"
+ ",ka.kyc_attributes_serial_id"
+ ",lo.jproperties::TEXT"
+ ",ka.collection_time"
+ ",ka.expiration_time"
+ ",ka.encrypted_attributes"
+ " FROM kyc_attributes ka"
+ " JOIN legitimization_processes lp"
+ " ON (ka.legitimization_serial = lp.legitimization_process_serial_id)"
+ " LEFT JOIN legitimization_outcomes lo"
+ " ON (ka.h_payto = lo.h_payto)"
+ /* **IF** we joined with 'lo', the lo must be active */
+ " WHERE COALESCE(lo.is_active,TRUE)"
+ " AND NOT ka.screened"
+ " ORDER BY ka.kyc_attributes_serial_id ASC"
+ );
+ qs = GNUNET_PQ_eval_prepared_multi_select (
+ pg->conn,
+ "iterate_unscreened_kyc_attributes",
+ params,
+ &get_attributes_cb,
+ &ctx);
+ if (GNUNET_OK != ctx.status)
+ return GNUNET_DB_STATUS_HARD_ERROR;
+ return qs;
+}
diff --git a/src/exchangedb/meson.build b/src/exchangedb/meson.build
@@ -197,7 +197,7 @@ libtalerexchangedb = library(
'get_aggregation_transient_by_wtid.c',
'get_open_legitimization_measure.c',
'get_pending_aggregation.c',
- 'iterate_all_kyc_attributes.c',
+ 'iterate_unscreened_kyc_attributes.c',
'iterate_all_purse_decisions_above_serial_id.c',
'iterate_all_purse_deletions_above_serial_id.c',
'iterate_aml_attributes.c',
@@ -242,6 +242,8 @@ libtalerexchangedb = library(
'iterate_withdrawals_above_serial_id.c',
'iterate_withdraw_amounts_for_kyc_check.c',
'update_to_aml_locked.c',
+ 'update_to_kyc_attributes_screened.c',
+ 'update_to_kyc_attributes_unscreened.c',
'start.c',
'start_deferred_wire_out.c',
'start_read_committed.c',
diff --git a/src/exchangedb/sql-schema/drop.sql b/src/exchangedb/sql-schema/drop.sql
@@ -17,13 +17,22 @@
-- Everything in one big transaction
BEGIN;
-WITH xpatches AS (
- SELECT patch_name
- FROM _v.patches
- WHERE starts_with(patch_name,'exchange-')
-)
- SELECT _v.unregister_patch(xpatches.patch_name)
- FROM xpatches;
+-- Later patches may require earlier ones, and _v.unregister_patch()
+-- refuses to remove a patch that another one still requires, so walk
+-- them from the newest to the oldest.
+DO $$
+DECLARE
+ r RECORD;
+BEGIN
+ FOR r IN
+ SELECT patch_name
+ FROM _v.patches
+ WHERE starts_with(patch_name,'exchange-')
+ ORDER BY patch_name DESC
+ LOOP
+ PERFORM _v.unregister_patch(r.patch_name);
+ END LOOP;
+END $$;
WITH xpatches AS (
SELECT patch_name
diff --git a/src/exchangedb/sql-schema/exchange-0018.sql b/src/exchangedb/sql-schema/exchange-0018.sql
@@ -0,0 +1,54 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2026 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+--
+
+BEGIN;
+
+SELECT _v.register_patch('exchange-0018', NULL, NULL);
+
+SET search_path TO exchange;
+
+CREATE FUNCTION alter_table_kyc_attributes18()
+RETURNS void
+LANGUAGE plpgsql
+AS $$
+BEGIN
+ ALTER TABLE kyc_attributes
+ ADD COLUMN screened BOOLEAN NOT NULL DEFAULT (FALSE);
+ COMMENT ON COLUMN kyc_attributes.screened
+ IS 'TRUE once taler-exchange-sanctionscheck screened these attributes against the current sanctions list; reset to FALSE whenever the sanctions list changes';
+ CREATE INDEX kyc_attributes_unscreened
+ ON kyc_attributes
+ (kyc_attributes_serial_id)
+ WHERE NOT screened;
+ COMMENT ON INDEX kyc_attributes_unscreened
+ IS 'used for iterate_unscreened_kyc_attributes';
+END
+$$;
+
+INSERT INTO exchange_tables
+ (name
+ ,version
+ ,action
+ ,partitioned
+ ,by_range)
+ VALUES
+ ('kyc_attributes18'
+ ,'exchange-0018'
+ ,'alter'
+ ,TRUE
+ ,FALSE);
+
+COMMIT;
diff --git a/src/exchangedb/sql-schema/meson.build b/src/exchangedb/sql-schema/meson.build
@@ -194,6 +194,7 @@ generated_sql = [
['exchange-0015.sql', ['exchange-0015.sql']],
['exchange-0016.sql', ['exchange-0016.sql']],
['exchange-0017.sql', ['exchange-0017.sql']],
+ ['exchange-0018.sql', ['exchange-0018.sql']],
['tops-0001.sql', ['tops-0001.sql']],
]
diff --git a/src/exchangedb/test_kyc_attributes.c b/src/exchangedb/test_kyc_attributes.c
@@ -21,7 +21,9 @@
*
* Covers #TALER_EXCHANGEDB_do_insert_kyc_attributes(),
* #TALER_EXCHANGEDB_iterate_kyc_attributes(),
- * #TALER_EXCHANGEDB_iterate_all_kyc_attributes(),
+ * #TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes(),
+ * #TALER_EXCHANGEDB_update_to_kyc_attributes_screened(),
+ * #TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened(),
* #TALER_EXCHANGEDB_iterate_aml_attributes(),
* #TALER_EXCHANGEDB_iterate_kyc_history(),
* #TALER_EXCHANGEDB_kyc_history_builder() and
@@ -42,10 +44,12 @@
#include "exchange-database/insert_aml_decision.h"
#include "exchange-database/insert_aml_officer.h"
#include "exchange-database/insert_legitimization_process.h"
-#include "exchange-database/iterate_all_kyc_attributes.h"
+#include "exchange-database/iterate_unscreened_kyc_attributes.h"
#include "exchange-database/iterate_aml_attributes.h"
#include "exchange-database/iterate_kyc_attributes.h"
#include "exchange-database/iterate_kyc_history.h"
+#include "exchange-database/update_to_kyc_attributes_screened.h"
+#include "exchange-database/update_to_kyc_attributes_unscreened.h"
/**
@@ -236,7 +240,7 @@ attribute_cb (void *cls,
/**
- * Callback for #TALER_EXCHANGEDB_iterate_all_kyc_attributes().
+ * Callback for #TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes().
*
* @param cls a `struct AttributeContext *`
* @param row_id row of the attributes
@@ -452,10 +456,10 @@ check_empty (struct TALER_EXCHANGEDB_PostgresContext *pg)
&attribute_cb,
&ctx));
FAILIF (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS !=
- TALER_EXCHANGEDB_iterate_all_kyc_attributes (pg,
- 0,
- &all_attributes_cb,
- &ctx));
+ TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ pg,
+ &all_attributes_cb,
+ &ctx));
FAILIF (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS !=
TALER_EXCHANGEDB_iterate_aml_attributes (pg,
&account.h_normalized,
@@ -528,6 +532,10 @@ check_insert (struct TALER_EXCHANGEDB_PostgresContext *pg)
"FROM kyc_attributes"
" WHERE NOT by_aml_officer"
" AND form_name='test-form'"));
+ /* new attributes still have to be screened against the sanctions list */
+ FAILIF (1 != TDB_count (pg,
+ "FROM kyc_attributes"
+ " WHERE NOT screened"));
/* the KYC process the attributes came from is now finished */
FAILIF (1 != TDB_count (pg,
"FROM legitimization_processes"
@@ -631,15 +639,15 @@ check_iterate (struct TALER_EXCHANGEDB_PostgresContext *pg)
ac_free (&ctx));
ac_free (&ctx);
- /* the global iterator sees both rows, in ascending row order */
+ /* the unscreened iterator sees both rows, in ascending row order */
memset (&ctx,
0,
sizeof (ctx));
FAILIF (0 >=
- TALER_EXCHANGEDB_iterate_all_kyc_attributes (pg,
- 0,
- &all_attributes_cb,
- &ctx));
+ TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ pg,
+ &all_attributes_cb,
+ &ctx));
FAILIF_C (2 != ctx.total,
ac_free (&ctx));
FAILIF_C (0 != strcmp (ctx.name,
@@ -649,18 +657,56 @@ check_iterate (struct TALER_EXCHANGEDB_PostgresContext *pg)
uint64_t first = ctx.row_id - 1;
ac_free (&ctx);
- /* ... and honours the minimum row ID */
+ /* ... but no longer the first row once that one is screened */
+ FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
+ TALER_EXCHANGEDB_update_to_kyc_attributes_screened (
+ pg,
+ &account.h_normalized,
+ first));
memset (&ctx,
0,
sizeof (ctx));
FAILIF (0 >=
- TALER_EXCHANGEDB_iterate_all_kyc_attributes (pg,
- first,
- &all_attributes_cb,
- &ctx));
+ TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ pg,
+ &all_attributes_cb,
+ &ctx));
FAILIF_C (1 != ctx.total,
ac_free (&ctx));
+ FAILIF_C (first + 1 != ctx.row_id,
+ ac_free (&ctx));
+ ac_free (&ctx);
+ /* marking a row under another account does nothing */
+ FAILIF (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS !=
+ TALER_EXCHANGEDB_update_to_kyc_attributes_screened (
+ pg,
+ &other,
+ first + 1));
+ /* with the second row screened as well, nothing is left */
+ FAILIF (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT !=
+ TALER_EXCHANGEDB_update_to_kyc_attributes_screened (
+ pg,
+ &account.h_normalized,
+ first + 1));
+ memset (&ctx,
+ 0,
+ sizeof (ctx));
+ FAILIF (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS !=
+ TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ pg,
+ &all_attributes_cb,
+ &ctx));
+ FAILIF_C (0 != ctx.total,
+ ac_free (&ctx));
ac_free (&ctx);
+ /* a new sanctions list makes both rows show up again */
+ FAILIF (2 !=
+ TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened (pg));
+ FAILIF (2 != TDB_count (pg,
+ "FROM kyc_attributes"
+ " WHERE NOT screened"));
+ FAILIF (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS !=
+ TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened (pg));
}
/* a callback that says stop is not called again */
@@ -669,10 +715,10 @@ check_iterate (struct TALER_EXCHANGEDB_PostgresContext *pg)
sizeof (ctx));
ctx.stop_after = 1;
FAILIF (0 >=
- TALER_EXCHANGEDB_iterate_all_kyc_attributes (pg,
- 0,
- &all_attributes_cb,
- &ctx));
+ TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ pg,
+ &all_attributes_cb,
+ &ctx));
FAILIF_C (1 != ctx.total,
ac_free (&ctx));
FAILIF_C (0 != strcmp (ctx.name,
diff --git a/src/exchangedb/test_legitimization_outcomes.c b/src/exchangedb/test_legitimization_outcomes.c
@@ -355,12 +355,14 @@ check_sanction_hit (struct TALER_EXCHANGEDB_PostgresContext *pg)
TALER_EXCHANGEDB_insert_sanction_list_hit (
pg,
&account.h_normalized,
+ 1,
true,
new_rules,
properties,
1,
events),
- json_decref (new_rules); json_decref (properties));
+ json_decref (new_rules);
+ json_decref (properties));
/* the previous outcome was deactivated, not deleted */
FAILIF_C (2 != TDB_count (pg,
"FROM legitimization_outcomes"),
diff --git a/src/exchangedb/update_to_kyc_attributes_screened.c b/src/exchangedb/update_to_kyc_attributes_screened.c
@@ -0,0 +1,49 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2026 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+/**
+ * @file exchangedb/update_to_kyc_attributes_screened.c
+ * @brief Implementation of the update_to_kyc_attributes_screened function for Postgres
+ * @author Christian Grothoff
+ */
+#include "taler/taler_pq_lib.h"
+#include "exchange-database/update_to_kyc_attributes_screened.h"
+#include "helper.h"
+
+
+enum GNUNET_DB_QueryStatus
+TALER_EXCHANGEDB_update_to_kyc_attributes_screened (
+ struct TALER_EXCHANGEDB_PostgresContext *pg,
+ const struct TALER_NormalizedPaytoHashP *h_payto,
+ uint64_t row_id)
+{
+ struct GNUNET_PQ_QueryParam params[] = {
+ GNUNET_PQ_query_param_auto_from_type (h_payto),
+ GNUNET_PQ_query_param_uint64 (&row_id),
+ GNUNET_PQ_query_param_end
+ };
+
+ /* h_payto is the partition key, row_id alone identifies the row */
+ PREPARE (pg,
+ "update_to_kyc_attributes_screened",
+ "UPDATE kyc_attributes"
+ " SET screened=TRUE"
+ " WHERE h_payto=$1"
+ " AND kyc_attributes_serial_id=$2;");
+ return GNUNET_PQ_eval_prepared_non_select (
+ pg->conn,
+ "update_to_kyc_attributes_screened",
+ params);
+}
diff --git a/src/exchangedb/update_to_kyc_attributes_unscreened.c b/src/exchangedb/update_to_kyc_attributes_unscreened.c
@@ -0,0 +1,43 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2026 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+/**
+ * @file exchangedb/update_to_kyc_attributes_unscreened.c
+ * @brief Implementation of the update_to_kyc_attributes_unscreened function for Postgres
+ * @author Christian Grothoff
+ */
+#include "taler/taler_pq_lib.h"
+#include "exchange-database/update_to_kyc_attributes_unscreened.h"
+#include "helper.h"
+
+
+enum GNUNET_DB_QueryStatus
+TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened (
+ struct TALER_EXCHANGEDB_PostgresContext *pg)
+{
+ struct GNUNET_PQ_QueryParam params[] = {
+ GNUNET_PQ_query_param_end
+ };
+
+ PREPARE (pg,
+ "update_to_kyc_attributes_unscreened",
+ "UPDATE kyc_attributes"
+ " SET screened=FALSE"
+ " WHERE screened;");
+ return GNUNET_PQ_eval_prepared_non_select (
+ pg->conn,
+ "update_to_kyc_attributes_unscreened",
+ params);
+}
diff --git a/src/include/exchange-database/insert_sanction_list_hit.h b/src/include/exchange-database/insert_sanction_list_hit.h
@@ -32,6 +32,7 @@
*
* @param pg the database context
* @param h_payto account for which the hit is to be stored
+ * @param kyc_attributes_serial_id which attributes were investigated
* @param to_investigate true to flag account for investigation,
* false to **preserve** existing status
* @param new_rules new KYC rules to apply to the account, NULL to preserve
@@ -42,15 +43,14 @@
* @return database transaction status
*/
enum GNUNET_DB_QueryStatus
-TALER_EXCHANGEDB_insert_sanction_list_hit (struct
- TALER_EXCHANGEDB_PostgresContext *pg,
- const struct
- TALER_NormalizedPaytoHashP *
- h_payto,
- bool to_investigate,
- const json_t *new_rules,
- const json_t *account_properties,
- unsigned int num_events,
- const char **events);
+TALER_EXCHANGEDB_insert_sanction_list_hit (
+ struct TALER_EXCHANGEDB_PostgresContext *pg,
+ const struct TALER_NormalizedPaytoHashP *h_payto,
+ uint64_t kyc_attributes_serial_id,
+ bool to_investigate,
+ const json_t *new_rules,
+ const json_t *account_properties,
+ unsigned int num_events,
+ const char **events);
#endif
diff --git a/src/include/exchange-database/iterate_all_kyc_attributes.h b/src/include/exchange-database/iterate_all_kyc_attributes.h
@@ -1,111 +0,0 @@
-/*
- This file is part of TALER
- Copyright (C) 2025 Taler Systems SA
-
- TALER is free software; you can redistribute it and/or modify it under the
- terms of the GNU General Public License as published by the Free Software
- Foundation; either version 3, or (at your option) any later version.
-
- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
-
- You should have received a copy of the GNU General Public License along with
- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
- */
-/**
- * @file src/include/exchange-database/iterate_all_kyc_attributes.h
- * @brief implementation of the iterate_all_kyc_attributes function for Postgres
- * @author Christian Grothoff
- */
-#ifndef EXCHANGE_DATABASE_ITERATE_ALL_KYC_ATTRIBUTES_H
-#define EXCHANGE_DATABASE_ITERATE_ALL_KYC_ATTRIBUTES_H
-
-#include "taler/taler_util.h"
-#include "taler/taler_json_lib.h"
-#include "exchangedb_lib.h"
-
-
-/**
- * Return KYC attribute information.
- *
- * @param cls closure
- * @param row_id current row in kyc_attributes table
- * @param h_payto account the attributes are about
- * @param provider_name name of the provider that collected the attributes
- * @param collection_time when were the attributes collected
- * @param expiration_time when does the data expire
- * @param properties properties that were set for @a h_payto
- * @param enc_attributes_size size of @a enc_attributes
- * @param enc_attributes the encrypted collected attributes
- * @return true to continue to iterate
- */
-#ifndef TALER_EXCHANGEDB_ALL_ATTRIBUTES_RESULT_CLOSURE
-/**
- * Type of the closure for #TALER_EXCHANGEDB_AllAttributesCallback.
- */
-#define TALER_EXCHANGEDB_ALL_ATTRIBUTES_RESULT_CLOSURE void
-#endif
-typedef bool
-(*TALER_EXCHANGEDB_AllAttributesCallback)(
- TALER_EXCHANGEDB_ALL_ATTRIBUTES_RESULT_CLOSURE *cls,
- uint64_t row_id,
- const struct TALER_NormalizedPaytoHashP *h_payto,
- const char *provider_name,
- struct GNUNET_TIME_Timestamp collection_time,
- struct GNUNET_TIME_Timestamp expiration_time,
- const json_t *properties,
- size_t enc_attributes_size,
- const void *enc_attributes);
-
-
-/* Callback typedefs */
-/**
- * Return KYC attribute information.
- *
- * @param cls closure
- * @param row_id current row in kyc_attributes table
- * @param h_payto account the attributes are about
- * @param provider_name name of the provider that collected the attributes
- * @param collection_time when were the attributes collected
- * @param expiration_time when does the data expire
- * @param properties properties that were set for @a h_payto
- * @param enc_attributes_size size of @a enc_attributes
- * @param enc_attributes the encrypted collected attributes
- * @return true to continue to iterate
- */
-typedef bool
-(*TALER_EXCHANGEDB_AllAttributesCallback)(
- TALER_EXCHANGEDB_ALL_ATTRIBUTES_RESULT_CLOSURE *cls,
- uint64_t row_id,
- const struct TALER_NormalizedPaytoHashP *h_payto,
- const char *provider_name,
- struct GNUNET_TIME_Timestamp collection_time,
- struct GNUNET_TIME_Timestamp expiration_time,
- const json_t *properties,
- size_t enc_attributes_size,
- const void *enc_attributes);
-
-/**
- * Lookup all KYC attributes above @a min_row_id.
- *
- * Primary test table: `kyc_attributes` (see test_kyc_attributes.c).
- *
- * @param pg the database context
- * @param min_row_id minimum row ID to return (exclusive)
- * @param cb callback to invoke on each match
- * @param cb_cls closure for @a cb
- * @return database transaction status
- */
-enum GNUNET_DB_QueryStatus
-TALER_EXCHANGEDB_iterate_all_kyc_attributes (struct
- TALER_EXCHANGEDB_PostgresContext *
- pg
- ,
- uint64_t min_row_id,
- TALER_EXCHANGEDB_AllAttributesCallback
- cb,
- TALER_EXCHANGEDB_ALL_ATTRIBUTES_RESULT_CLOSURE
- *cb_cls);
-
-#endif
diff --git a/src/include/exchange-database/iterate_unscreened_kyc_attributes.h b/src/include/exchange-database/iterate_unscreened_kyc_attributes.h
@@ -0,0 +1,83 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2025 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+/**
+ * @file src/include/exchange-database/iterate_unscreened_kyc_attributes.h
+ * @brief implementation of the iterate_unscreened_kyc_attributes function for Postgres
+ * @author Christian Grothoff
+ */
+#ifndef EXCHANGE_DATABASE_ITERATE_UNSCREENED_KYC_ATTRIBUTES_H
+#define EXCHANGE_DATABASE_ITERATE_UNSCREENED_KYC_ATTRIBUTES_H
+
+#include "taler/taler_util.h"
+#include "taler/taler_json_lib.h"
+#include "exchangedb_lib.h"
+
+
+#ifndef TALER_EXCHANGEDB_UNSCREENED_ATTRIBUTES_RESULT_CLOSURE
+/**
+ * Type of the closure for #TALER_EXCHANGEDB_UnscreenedAttributesCallback.
+ */
+#define TALER_EXCHANGEDB_UNSCREENED_ATTRIBUTES_RESULT_CLOSURE void
+#endif
+
+/**
+ * Return KYC attribute information.
+ *
+ * @param cls closure
+ * @param row_id current row in kyc_attributes table
+ * @param h_payto account the attributes are about
+ * @param provider_name name of the provider that collected the attributes
+ * @param collection_time when were the attributes collected
+ * @param expiration_time when does the data expire
+ * @param properties properties that were set for @a h_payto
+ * @param enc_attributes_size size of @a enc_attributes
+ * @param enc_attributes the encrypted collected attributes
+ * @return true to continue to iterate
+ */
+typedef bool
+(*TALER_EXCHANGEDB_UnscreenedAttributesCallback)(
+ TALER_EXCHANGEDB_UNSCREENED_ATTRIBUTES_RESULT_CLOSURE *cls,
+ uint64_t row_id,
+ const struct TALER_NormalizedPaytoHashP *h_payto,
+ const char *provider_name,
+ struct GNUNET_TIME_Timestamp collection_time,
+ struct GNUNET_TIME_Timestamp expiration_time,
+ const json_t *properties,
+ size_t enc_attributes_size,
+ const void *enc_attributes);
+
+
+/**
+ * Lookup all KYC attributes that have not yet been screened against the
+ * current sanctions list, in ascending row order. Rows are marked as
+ * screened with #TALER_EXCHANGEDB_update_to_kyc_attributes_screened(), and
+ * #TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened() makes all rows
+ * show up here again.
+ *
+ * Primary test table: `kyc_attributes` (see test_kyc_attributes.c).
+ *
+ * @param pg the database context
+ * @param cb callback to invoke on each match
+ * @param cb_cls closure for @a cb
+ * @return database transaction status
+ */
+enum GNUNET_DB_QueryStatus
+TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes (
+ struct TALER_EXCHANGEDB_PostgresContext *pg,
+ TALER_EXCHANGEDB_UnscreenedAttributesCallback cb,
+ TALER_EXCHANGEDB_UNSCREENED_ATTRIBUTES_RESULT_CLOSURE *cb_cls);
+
+#endif
diff --git a/src/include/exchange-database/update_to_kyc_attributes_screened.h b/src/include/exchange-database/update_to_kyc_attributes_screened.h
@@ -0,0 +1,49 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2026 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+/**
+ * @file src/include/exchange-database/update_to_kyc_attributes_screened.h
+ * @brief implementation of the update_to_kyc_attributes_screened function for Postgres
+ * @author Christian Grothoff
+ */
+#ifndef EXCHANGE_DATABASE_UPDATE_TO_KYC_ATTRIBUTES_SCREENED_H
+#define EXCHANGE_DATABASE_UPDATE_TO_KYC_ATTRIBUTES_SCREENED_H
+
+#include "taler/taler_util.h"
+#include "exchangedb_lib.h"
+
+
+/**
+ * Mark the KYC attributes in row @a row_id as screened against the
+ * current sanctions list, so that
+ * #TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes() no longer returns
+ * them.
+ *
+ * Primary test table: `kyc_attributes` (see test_kyc_attributes.c).
+ *
+ * @param pg the database context
+ * @param h_payto account the attributes are about
+ * @param row_id row of the attributes in the kyc_attributes table
+ * @return transaction status; #GNUNET_DB_STATUS_SUCCESS_NO_RESULTS if
+ * the row no longer exists
+ */
+enum GNUNET_DB_QueryStatus
+TALER_EXCHANGEDB_update_to_kyc_attributes_screened (
+ struct TALER_EXCHANGEDB_PostgresContext *pg,
+ const struct TALER_NormalizedPaytoHashP *h_payto,
+ uint64_t row_id);
+
+
+#endif
diff --git a/src/include/exchange-database/update_to_kyc_attributes_unscreened.h b/src/include/exchange-database/update_to_kyc_attributes_unscreened.h
@@ -0,0 +1,44 @@
+/*
+ This file is part of TALER
+ Copyright (C) 2026 Taler Systems SA
+
+ TALER is free software; you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation; either version 3, or (at your option) any later version.
+
+ TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+ A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along with
+ TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+ */
+/**
+ * @file src/include/exchange-database/update_to_kyc_attributes_unscreened.h
+ * @brief implementation of the update_to_kyc_attributes_unscreened function for Postgres
+ * @author Christian Grothoff
+ */
+#ifndef EXCHANGE_DATABASE_UPDATE_TO_KYC_ATTRIBUTES_UNSCREENED_H
+#define EXCHANGE_DATABASE_UPDATE_TO_KYC_ATTRIBUTES_UNSCREENED_H
+
+#include "taler/taler_util.h"
+#include "exchangedb_lib.h"
+
+
+/**
+ * Mark all KYC attributes as not yet screened against the sanctions list,
+ * so that #TALER_EXCHANGEDB_iterate_unscreened_kyc_attributes() returns
+ * all of them again. Used when the sanctions list changed.
+ *
+ * Primary test table: `kyc_attributes` (see test_kyc_attributes.c).
+ *
+ * @param pg the database context
+ * @return transaction status, the number of rows that were marked as
+ * screened before on success
+ */
+enum GNUNET_DB_QueryStatus
+TALER_EXCHANGEDB_update_to_kyc_attributes_unscreened (
+ struct TALER_EXCHANGEDB_PostgresContext *pg);
+
+
+#endif
