commit 104d95a77a3e39f7abc0a8b8e877bd60e2a511a2
parent 61139f478348640dc5e11d2ad4f55f473216256e
Author: Christian Grothoff <christian@grothoff.org>
Date: Tue, 3 Jun 2025 16:18:19 +0200
make limits in taler-exchange-sanctionscheck configurable, support incremental runs of the tool and add support for background mode (work on #9053)
Diffstat:
2 files changed, 318 insertions(+), 28 deletions(-)
diff --git a/src/exchange/exchange.conf b/src/exchange/exchange.conf
@@ -129,3 +129,26 @@ PRIVACY_DIR = ${TALER_DATA_HOME}terms/
# Etag / filename for the privacy policy.
PRIVACY_ETAG = exchange-pp-v0
+
+
+[exchange-sanctionscheck]
+# Name where we store the sanctions check offset.
+MIN_ROW_FILENAME = ${TALER_CACHE_HOME}sanctionscheck-offset.bin
+
+# Sanction list match rating that must be exceeded for an automated
+# freeze of the account (without manual investigation first).
+# Both this and the FREEZE_CONFIDENCE_LIMIT must be met.
+FREEZE_RATING_LIMIT = 0.95
+
+# Sanction list confidence that must be exceeded for an automated
+# freeze of the account (without manual investigation first).
+# Both this and the FREEZE_RATING_LIMIT must be met.
+FREEZE_CONFIDENCE_LIMIT = 0.95
+
+# Value that the rating divided by the confidence must exceed to
+# trigger a (manual) investigation. At 0.95 (default), a modest
+# 0.8 match with low 0.5 confidence will trigger (1.6), but a
+# good 0.9 match with a super-high 1.0 confidence would not (0.9 < 0.95).
+# OTOH, a bad 0.2 match with super-low 0.1 confidence would again
+# trigger (2.0 > 0.95).
+INVESTIGATION_LIMIT = 0.95
+\ No newline at end of file
diff --git a/src/exchange/taler-exchange-sanctionscheck.c b/src/exchange/taler-exchange-sanctionscheck.c
@@ -23,6 +23,7 @@
#include <jansson.h>
#include <pthread.h>
#include <microhttpd.h>
+#include "taler_dbevents.h"
#include "taler_exchangedb_lib.h"
#include "taler_exchangedb_plugin.h"
#include "taler_json_lib.h"
@@ -115,6 +116,82 @@ static struct Account *acc_tail;
static uint64_t min_row_id;
/**
+ * File descriptor with the name of the file where we track our
+ * progress.
+ */
+static int min_row_fd = -1;
+
+/**
+ * '-t' command line flag. Disables background processing mode.
+ */
+static int testmode;
+
+/**
+ * '-r' command line flag. Restarts analysis from scratch (for
+ * fresh sanction list).
+ */
+static int reset;
+
+/**
+ * '-n' command line flag. Do not actually run, only reset.
+ */
+static int norun;
+
+/**
+ * Handler to learn about updated KYC attributes.
+ */
+static struct GNUNET_DB_EventHandler *eh;
+
+/**
+ * Set to true if we should restart immediately after
+ * finishing the current transaction.
+ */
+static bool restart_now;
+
+/**
+ * Set to true while we are in a database transaction iterating
+ * over KYC attributes.
+ */
+static bool in_transaction;
+
+/**
+ * Match quality needed for instantly freezing an account.
+ */
+static float freeze_rating_limit = 0.95;
+
+/**
+ * Match confidence needed for instantly freezing an account.
+ */
+static float freeze_confidence_limit = 0.95;
+
+/**
+ * Rating/confidence threshold that must be passed to begin
+ * an investigation.
+ */
+static float investigation_limit = 0.95;
+
+/**
+ * Write @a min_row_id to @a min_row_fd.
+ */
+static void
+sync_row (void)
+{
+ uint64_t r = GNUNET_htonll (min_row_id);
+
+ GNUNET_break (-1 != min_row_fd);
+ GNUNET_break (0 == lseek (min_row_fd,
+ 0,
+ SEEK_SET));
+ GNUNET_break (sizeof (r) ==
+ write (min_row_fd,
+ &r,
+ sizeof (r)));
+ GNUNET_break (0 ==
+ fsync (min_row_fd));
+}
+
+
+/**
* We're being aborted with CTRL-C (or SIGTERM). Shut down.
*
* @param cls closure
@@ -125,6 +202,12 @@ shutdown_task (void *cls)
struct Account *acc;
(void) cls;
+ sync_row ();
+ if (-1 != min_row_fd)
+ {
+ GNUNET_break (0 == close (min_row_fd));
+ min_row_fd = -1;
+ }
while (NULL != (acc = acc_head))
{
GNUNET_CONTAINER_DLL_remove (acc_head,
@@ -133,7 +216,12 @@ shutdown_task (void *cls)
json_decref (acc->properties);
GNUNET_free (acc);
}
-
+ if (NULL != eh)
+ {
+ db_plugin->event_listen_cancel (db_plugin->cls,
+ eh);
+ eh = NULL;
+ }
if (NULL != sr)
{
TALER_KYCLOGIC_sanction_rater_stop (sr);
@@ -164,6 +252,13 @@ double_to_billion (double d)
/**
+ * Start the actual database transaction.
+ */
+static void
+begin_transaction (void);
+
+
+/**
* Function called with the result of a sanction evaluation.
*
* @param cls closure
@@ -185,14 +280,13 @@ sanction_cb (void *cls,
bool freeze = false;
bool investigate = false;
- // FIXME-#9053: formulas to be improved and customized via configuration
- if ( (rating > 0.95) &&
- (confidence > 0.95) )
+ if ( (rating > (double) freeze_rating_limit) &&
+ (confidence > (double) freeze_confidence_limit) )
{
freeze = true;
investigate = true;
}
- else if (rating > 0.95 - confidence)
+ else if (rating > (double) investigation_limit * confidence)
{
investigate = true;
}
@@ -247,6 +341,7 @@ sanction_cb (void *cls,
acc_tail,
acc);
json_decref (acc->properties);
+ min_row_id = acc->row_id;
GNUNET_free (acc);
if (NULL != acc_head)
return; /* more work */
@@ -263,7 +358,15 @@ sanction_cb (void *cls,
return;
}
}
- GNUNET_SCHEDULER_shutdown ();
+ in_transaction = false;
+ sync_row ();
+ if (restart_now)
+ {
+ begin_transaction ();
+ return;
+ }
+ if (testmode)
+ GNUNET_SCHEDULER_shutdown ();
}
@@ -404,6 +507,76 @@ init_freeze (void)
}
+static void
+begin_transaction ()
+{
+ enum GNUNET_DB_QueryStatus qs;
+
+ restart_now = false;
+ GNUNET_assert (! in_transaction);
+ if (GNUNET_OK !=
+ db_plugin->start (db_plugin->cls,
+ "sanctionscheck"))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ "Failed to begin DB transaction\n");
+ global_ret = EXIT_NOTCONFIGURED;
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ in_transaction = true;
+ /* FIXME-10063: we may want to eventually limit the number of
+ records we process in a single transaction. */
+ qs = db_plugin->select_all_kyc_attributes (db_plugin->cls,
+ min_row_id,
+ &account_cb,
+ NULL);
+ if (qs < 0)
+ {
+ global_ret = EXIT_FAILURE;
+ GNUNET_break (0);
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+ {
+ db_plugin->rollback (db_plugin->cls);
+ GNUNET_SCHEDULER_shutdown ();
+ return;
+ }
+ if (NULL == acc_head)
+ in_transaction = false;
+ if ( (NULL == acc_head) &&
+ (testmode) )
+ {
+ /* no work, not incremental, we are done */
+ GNUNET_SCHEDULER_shutdown ();
+ }
+}
+
+
+/**
+ * Function called on new KYC attributes being available in Postgres.
+ *
+ * @param cls closure
+ * @param extra additional event data provided
+ * @param extra_size number of bytes in @a extra
+ */
+static void
+db_event_cb (void *cls,
+ const void *extra,
+ size_t extra_size)
+{
+ GNUNET_break (NULL == cls);
+ (void) extra;
+ (void) extra_size;
+ if (in_transaction)
+ restart_now = true;
+ else
+ begin_transaction ();
+}
+
+
/**
* First task.
*
@@ -418,8 +591,6 @@ run (void *cls,
const char *cfgfile,
const struct GNUNET_CONFIGURATION_Handle *c)
{
- enum GNUNET_DB_QueryStatus qs;
-
(void) cls;
(void) cfgfile;
cfg = c;
@@ -431,6 +602,42 @@ run (void *cls,
GNUNET_SCHEDULER_shutdown ();
return;
}
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_float (cfg,
+ "exchange-sanctionscheck",
+ "FREEZE_RATING_LIMIT",
+ &freeze_rating_limit))
+ {
+ GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+ "exchange-sanctionscheck",
+ "FREEZE_RATING_LIMIT");
+ global_ret = EXIT_NOTCONFIGURED;
+ return;
+ }
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_float (cfg,
+ "exchange-sanctionscheck",
+ "FREEZE_CONFIDENCE_LIMIT",
+ &freeze_confidence_limit))
+ {
+ GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+ "exchange-sanctionscheck",
+ "FREEZE_CONFIDENCE_LIMIT");
+ global_ret = EXIT_NOTCONFIGURED;
+ return;
+ }
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_float (cfg,
+ "exchange-sanctionscheck",
+ "INVESTIGATION_LIMIT",
+ &investigation_limit))
+ {
+ GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+ "exchange-sanctionscheck",
+ "INVESTIGATION_LIMIT");
+ global_ret = EXIT_NOTCONFIGURED;
+ return;
+ }
if (! init_freeze ())
return;
{
@@ -472,34 +679,81 @@ run (void *cls,
GNUNET_SCHEDULER_shutdown ();
return;
}
- if (GNUNET_OK !=
- db_plugin->start (db_plugin->cls,
- "sanctionscheck"))
{
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- "Failed to begin DB transaction\n");
- global_ret = EXIT_NOTCONFIGURED;
- GNUNET_SCHEDULER_shutdown ();
- return;
+ char *min_row_fn;
+ uint64_t r;
+
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_string (cfg,
+ "exchange-sanctionscheck",
+ "MIN_ROW_FILENAME",
+ &min_row_fn))
+ {
+ GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
+ "exchange-sanctionscheck",
+ "MIN_ROW_FILENAME");
+ global_ret = EXIT_NOTCONFIGURED;
+ return;
+ }
+ if (reset &&
+ (0 != unlink (min_row_fn)) &&
+ (ENOENT != errno) )
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "unlink",
+ min_row_fn);
+ GNUNET_free (min_row_fn);
+ global_ret = EXIT_NOPERMISSION;
+ return;
+ }
+ min_row_fd = open (min_row_fn,
+ O_CREAT,
+ S_IRUSR | S_IWUSR);
+ if (-1 == min_row_fd)
+ {
+ GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
+ "open",
+ min_row_fn);
+ GNUNET_free (min_row_fn);
+ global_ret = EXIT_NOTCONFIGURED;
+ return;
+ }
+ if (sizeof (r) !=
+ read (min_row_fd,
+ &r,
+ sizeof (r)))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Could not read starting row from `%s', will start from 0\n",
+ min_row_fn);
+ min_row_id = 0;
+ }
+ else
+ {
+ min_row_id = GNUNET_ntohll (r);
+ }
+ GNUNET_free (min_row_fn);
}
- qs = db_plugin->select_all_kyc_attributes (db_plugin->cls,
- min_row_id,
- &account_cb,
- NULL);
- if (qs < 0)
+ if (norun)
{
- global_ret = EXIT_FAILURE;
- GNUNET_break (0);
GNUNET_SCHEDULER_shutdown ();
return;
}
- if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
+ if (! testmode)
{
- db_plugin->rollback (db_plugin->cls);
- GNUNET_SCHEDULER_shutdown ();
- return;
+ struct GNUNET_DB_EventHeaderP hdr = {
+ .size = htons (sizeof (hdr)),
+ .type = htons (TALER_DBEVENT_EXCHANGE_NEW_KYC_ATTRIBUTES),
+ };
+
+ eh = db_plugin->event_listen (
+ db_plugin->cls,
+ GNUNET_TIME_UNIT_FOREVER_REL,
+ &hdr,
+ &db_event_cb,
+ NULL);
}
- GNUNET_assert (NULL != acc_head);
+ begin_transaction ();
}
@@ -516,6 +770,18 @@ main (int argc,
{
struct GNUNET_GETOPT_CommandLineOption options[] = {
GNUNET_GETOPT_option_version (VERSION "-" VCS_VERSION),
+ GNUNET_GETOPT_option_flag ('n',
+ "norun",
+ "do not actually start a scan (to be used to only reset without starting a scan)",
+ &reset),
+ GNUNET_GETOPT_option_flag ('r',
+ "reset",
+ "rescan all records (to be used when the sanction list was updated)",
+ &reset),
+ GNUNET_GETOPT_option_flag ('t',
+ "test",
+ "run in test mode and exit when idle",
+ &testmode),
GNUNET_GETOPT_OPTION_END
};
enum GNUNET_GenericReturnValue ret;
