From 2ace9969b7e1ede610ff99546c5a84f59adf0931 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sat, 21 Mar 2020 11:05:51 +0100 Subject: rename fest on refactored auditor logic --- src/auditor/.gitignore | 5 + src/auditor/Makefile.am | 64 +- src/auditor/report-lib.c | 243 +-- src/auditor/report-lib.h | 85 +- src/auditor/taler-auditor-aggregation.c | 1511 --------------- src/auditor/taler-auditor-coins.c | 2346 ----------------------- src/auditor/taler-auditor-deposits.c | 360 ---- src/auditor/taler-auditor-reserves.c | 1641 ---------------- src/auditor/taler-helper-auditor-aggregation.c | 1548 +++++++++++++++ src/auditor/taler-helper-auditor-coins.c | 2374 ++++++++++++++++++++++++ src/auditor/taler-helper-auditor-deposits.c | 369 ++++ src/auditor/taler-helper-auditor-reserves.c | 1674 +++++++++++++++++ src/auditor/taler-helper-auditor-wire.c | 2229 ++++++++++++++++++++++ src/auditor/taler-wire-auditor.c | 2354 ----------------------- src/auditor/test-auditor.sh | 22 +- 15 files changed, 8414 insertions(+), 8411 deletions(-) delete mode 100644 src/auditor/taler-auditor-aggregation.c delete mode 100644 src/auditor/taler-auditor-coins.c delete mode 100644 src/auditor/taler-auditor-deposits.c delete mode 100644 src/auditor/taler-auditor-reserves.c create mode 100644 src/auditor/taler-helper-auditor-aggregation.c create mode 100644 src/auditor/taler-helper-auditor-coins.c create mode 100644 src/auditor/taler-helper-auditor-deposits.c create mode 100644 src/auditor/taler-helper-auditor-reserves.c create mode 100644 src/auditor/taler-helper-auditor-wire.c delete mode 100644 src/auditor/taler-wire-auditor.c (limited to 'src') diff --git a/src/auditor/.gitignore b/src/auditor/.gitignore index fe067a536..d065d10a5 100644 --- a/src/auditor/.gitignore +++ b/src/auditor/.gitignore @@ -13,3 +13,8 @@ test-wire-audit-inc.json wirefees/ bank.err libauditor.a +taler-helper-auditor-aggregation +taler-helper-auditor-coins +taler-helper-auditor-deposits +taler-helper-auditor-reserves +taler-helper-auditor-wire diff --git a/src/auditor/Makefile.am b/src/auditor/Makefile.am index 819789efd..13410a16b 100644 --- a/src/auditor/Makefile.am +++ b/src/auditor/Makefile.am @@ -13,11 +13,11 @@ pkgcfg_DATA = \ bin_PROGRAMS = \ taler-auditor \ - taler-auditor-reserves \ - taler-auditor-coins \ - taler-auditor-aggregation \ - taler-auditor-deposits \ - taler-wire-auditor \ + taler-helper-auditor-reserves \ + taler-helper-auditor-coins \ + taler-helper-auditor-aggregation \ + taler-helper-auditor-deposits \ + taler-helper-auditor-wire \ taler-auditor-exchange \ taler-auditor-httpd \ taler-auditor-sign \ @@ -44,9 +44,9 @@ taler_auditor_dbinit_CPPFLAGS = \ -I$(top_srcdir)/src/pq/ \ $(POSTGRESQL_CPPFLAGS) -taler_auditor_reserves_SOURCES = \ - taler-auditor-reserves.c -taler_auditor_reserves_LDADD = \ +taler_helper_auditor_reserves_SOURCES = \ + taler-helper-auditor-reserves.c +taler_helper_auditor_reserves_LDADD = \ $(LIBGCRYPT_LIBS) \ $(top_builddir)/src/util/libtalerutil.la \ $(top_builddir)/src/json/libtalerjson.la \ @@ -58,9 +58,9 @@ taler_auditor_reserves_LDADD = \ -lgnunetjson \ -lgnunetutil -taler_auditor_coins_SOURCES = \ - taler-auditor-coins.c -taler_auditor_coins_LDADD = \ +taler_helper_auditor_coins_SOURCES = \ + taler-helper-auditor-coins.c +taler_helper_auditor_coins_LDADD = \ $(LIBGCRYPT_LIBS) \ $(top_builddir)/src/util/libtalerutil.la \ $(top_builddir)/src/json/libtalerjson.la \ @@ -72,9 +72,9 @@ taler_auditor_coins_LDADD = \ -lgnunetjson \ -lgnunetutil -taler_auditor_aggregation_SOURCES = \ - taler-auditor-aggregation.c -taler_auditor_aggregation_LDADD = \ +taler_helper_auditor_aggregation_SOURCES = \ + taler-helper-auditor-aggregation.c +taler_helper_auditor_aggregation_LDADD = \ $(LIBGCRYPT_LIBS) \ $(top_builddir)/src/util/libtalerutil.la \ $(top_builddir)/src/json/libtalerjson.la \ @@ -86,9 +86,9 @@ taler_auditor_aggregation_LDADD = \ -lgnunetjson \ -lgnunetutil -taler_auditor_deposits_SOURCES = \ - taler-auditor-deposits.c -taler_auditor_deposits_LDADD = \ +taler_helper_auditor_deposits_SOURCES = \ + taler-helper-auditor-deposits.c +taler_helper_auditor_deposits_LDADD = \ $(LIBGCRYPT_LIBS) \ $(top_builddir)/src/util/libtalerutil.la \ $(top_builddir)/src/json/libtalerjson.la \ @@ -100,6 +100,22 @@ taler_auditor_deposits_LDADD = \ -lgnunetjson \ -lgnunetutil +taler_helper_auditor_wire_SOURCES = \ + taler-helper-auditor-wire.c +taler_helper_auditor_wire_LDADD = \ + $(LIBGCRYPT_LIBS) \ + $(top_builddir)/src/util/libtalerutil.la \ + $(top_builddir)/src/json/libtalerjson.la \ + $(top_builddir)/src/bank-lib/libtalerbank.la \ + $(top_builddir)/src/exchangedb/libtalerexchangedb.la \ + $(top_builddir)/src/auditordb/libtalerauditordb.la \ + libauditor.a \ + -ljansson \ + -lgnunetjson \ + -lgnunetcurl \ + -lgnunetutil + + taler_auditor_SOURCES = \ taler-auditor.c taler_auditor_LDADD = \ @@ -131,20 +147,6 @@ taler_auditor_httpd_LDADD = \ -lgnunetutil \ -lz -taler_wire_auditor_SOURCES = \ - taler-wire-auditor.c -taler_wire_auditor_LDADD = \ - $(LIBGCRYPT_LIBS) \ - $(top_builddir)/src/util/libtalerutil.la \ - $(top_builddir)/src/json/libtalerjson.la \ - $(top_builddir)/src/bank-lib/libtalerbank.la \ - $(top_builddir)/src/exchangedb/libtalerexchangedb.la \ - $(top_builddir)/src/auditordb/libtalerauditordb.la \ - -ljansson \ - -lgnunetjson \ - -lgnunetcurl \ - -lgnunetutil - taler_auditor_sign_SOURCES = \ taler-auditor-sign.c taler_auditor_sign_LDADD = \ diff --git a/src/auditor/report-lib.c b/src/auditor/report-lib.c index b2df8a14d..b00a522a7 100644 --- a/src/auditor/report-lib.c +++ b/src/auditor/report-lib.c @@ -22,49 +22,49 @@ #include "report-lib.h" /** - * Command-line option "-r": restart audit from scratch + * Command-line option "-r": TALER_ARL_restart audit from scratch */ -int restart; +int TALER_ARL_restart; /** * Handle to access the exchange's database. */ -struct TALER_EXCHANGEDB_Plugin *edb; +struct TALER_EXCHANGEDB_Plugin *TALER_ARL_edb; /** - * Which currency are we doing the audit for? + * Which TALER_ARL_currency are we doing the audit for? */ -char *currency; +char *TALER_ARL_currency; /** - * How many fractional digits does the currency use? + * How many fractional digits does the TALER_ARL_currency use? */ -struct TALER_Amount currency_round_unit; +struct TALER_Amount TALER_ARL_currency_round_unit; /** * Our configuration. */ -const struct GNUNET_CONFIGURATION_Handle *cfg; +const struct GNUNET_CONFIGURATION_Handle *TALER_ARL_cfg; /** - * Our session with the #edb. + * Our session with the #TALER_ARL_edb. */ -struct TALER_EXCHANGEDB_Session *esession; +struct TALER_EXCHANGEDB_Session *TALER_ARL_esession; /** * Handle to access the auditor's database. */ -struct TALER_AUDITORDB_Plugin *adb; +struct TALER_AUDITORDB_Plugin *TALER_ARL_adb; /** - * Our session with the #adb. + * Our session with the #TALER_ARL_adb. */ -struct TALER_AUDITORDB_Session *asession; +struct TALER_AUDITORDB_Session *TALER_ARL_asession; /** * Master public key of the exchange to audit. */ -struct TALER_MasterPublicKeyP master_pub; +struct TALER_MasterPublicKeyP TALER_ARL_master_pub; /** * At what time did the auditor process start? @@ -85,7 +85,7 @@ static struct GNUNET_CONTAINER_MultiHashMap *denominations; * @return human-readable string representing the time */ json_t * -json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at) +TALER_ARL_TALER_ARL_json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at) { return json_string (GNUNET_STRINGS_absolute_time_to_string @@ -100,7 +100,7 @@ json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at) * @return human-readable string representing the time */ json_t * -json_from_time_abs (struct GNUNET_TIME_Absolute at) +TALER_ARL_json_from_time_abs (struct GNUNET_TIME_Absolute at) { return json_string (GNUNET_STRINGS_absolute_time_to_string (at)); @@ -108,14 +108,14 @@ json_from_time_abs (struct GNUNET_TIME_Absolute at) /** - * Add @a object to the report @a array. Fail hard if this fails. + * Add @a object to the TALER_ARL_report @a array. Fail hard if this fails. * - * @param array report array to append @a object to + * @param array TALER_ARL_report array to append @a object to * @param object object to append, should be check that it is not NULL */ void -report (json_t *array, - json_t *object) +TALER_ARL_report (json_t *array, + json_t *object) { GNUNET_assert (NULL != object); GNUNET_assert (0 == @@ -185,9 +185,9 @@ add_denomination (void *cls, * @return transaction status code */ enum GNUNET_DB_QueryStatus -get_denomination_info_by_hash (const struct GNUNET_HashCode *dh, - const struct - TALER_DenominationKeyValidityPS **issue) +TALER_ARL_get_denomination_info_by_hash (const struct GNUNET_HashCode *dh, + const struct + TALER_DenominationKeyValidityPS **issue) { const struct TALER_DenominationKeyValidityPS *i; enum GNUNET_DB_QueryStatus qs; @@ -196,11 +196,11 @@ get_denomination_info_by_hash (const struct GNUNET_HashCode *dh, { denominations = GNUNET_CONTAINER_multihashmap_create (256, GNUNET_NO); - qs = adb->select_denomination_info (adb->cls, - asession, - &master_pub, - &add_denomination, - NULL); + qs = TALER_ARL_adb->select_denomination_info (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &add_denomination, + NULL); if (0 > qs) { *issue = NULL; @@ -216,11 +216,11 @@ get_denomination_info_by_hash (const struct GNUNET_HashCode *dh, return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; } /* maybe database changed since we last iterated, give it one more shot */ - qs = adb->select_denomination_info (adb->cls, - asession, - &master_pub, - &add_denomination, - NULL); + qs = TALER_ARL_adb->select_denomination_info (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &add_denomination, + NULL); if (qs <= 0) { if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) @@ -255,10 +255,11 @@ get_denomination_info_by_hash (const struct GNUNET_HashCode *dh, * @return transaction status code */ enum GNUNET_DB_QueryStatus -get_denomination_info (const struct TALER_DenominationPublicKey *denom_pub, - const struct - TALER_DenominationKeyValidityPS **issue, - struct GNUNET_HashCode *dh) +TALER_ARL_get_denomination_info (const struct + TALER_DenominationPublicKey *denom_pub, + const struct + TALER_DenominationKeyValidityPS **issue, + struct GNUNET_HashCode *dh) { struct GNUNET_HashCode hc; @@ -266,8 +267,8 @@ get_denomination_info (const struct TALER_DenominationPublicKey *denom_pub, dh = &hc; GNUNET_CRYPTO_rsa_public_key_hash (denom_pub->rsa_public_key, dh); - return get_denomination_info_by_hash (dh, - issue); + return TALER_ARL_get_denomination_info_by_hash (dh, + issue); } @@ -281,25 +282,25 @@ get_denomination_info (const struct TALER_DenominationPublicKey *denom_pub, * #GNUNET_NO if we had an error on commit (retry may help) * #GNUNET_SYSERR on hard errors */ -int -transact (Analysis analysis, +static int +transact (TALER_ARL_Analysis analysis, void *analysis_cls) { int ret; enum GNUNET_DB_QueryStatus qs; - ret = adb->start (adb->cls, - asession); + ret = TALER_ARL_adb->start (TALER_ARL_adb->cls, + TALER_ARL_asession); if (GNUNET_OK != ret) { GNUNET_break (0); return GNUNET_SYSERR; } - edb->preflight (edb->cls, - esession); - ret = edb->start (edb->cls, - esession, - "auditor"); + TALER_ARL_edb->preflight (TALER_ARL_edb->cls, + TALER_ARL_esession); + ret = TALER_ARL_edb->start (TALER_ARL_edb->cls, + TALER_ARL_esession, + "auditor"); if (GNUNET_OK != ret) { GNUNET_break (0); @@ -308,20 +309,20 @@ transact (Analysis analysis, qs = analysis (analysis_cls); if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) { - qs = edb->commit (edb->cls, - esession); + qs = TALER_ARL_edb->commit (TALER_ARL_edb->cls, + TALER_ARL_esession); if (0 > qs) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Exchange DB commit failed, rolling back transaction\n"); - adb->rollback (adb->cls, - asession); + TALER_ARL_adb->rollback (TALER_ARL_adb->cls, + TALER_ARL_asession); } else { - qs = adb->commit (adb->cls, - asession); + qs = TALER_ARL_adb->commit (TALER_ARL_adb->cls, + TALER_ARL_asession); if (0 > qs) { GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); @@ -334,10 +335,10 @@ transact (Analysis analysis, { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Processing failed (or no changes), rolling back transaction\n"); - adb->rollback (adb->cls, - asession); - edb->rollback (edb->cls, - esession); + TALER_ARL_adb->rollback (TALER_ARL_adb->cls, + TALER_ARL_asession); + TALER_ARL_edb->rollback (TALER_ARL_edb->cls, + TALER_ARL_esession); } switch (qs) { @@ -362,18 +363,18 @@ transact (Analysis analysis, * @return #GNUNET_OK on success */ int -setup_sessions_and_run (Analysis ana, - void *ana_cls) +TALER_ARL_setup_sessions_and_run (TALER_ARL_Analysis ana, + void *ana_cls) { - esession = edb->get_session (edb->cls); - if (NULL == esession) + TALER_ARL_esession = TALER_ARL_edb->get_session (TALER_ARL_edb->cls); + if (NULL == TALER_ARL_esession) { fprintf (stderr, "Failed to initialize exchange session.\n"); return GNUNET_SYSERR; } - asession = adb->get_session (adb->cls); - if (NULL == asession) + TALER_ARL_asession = TALER_ARL_adb->get_session (TALER_ARL_adb->cls); + if (NULL == TALER_ARL_asession) { fprintf (stderr, "Failed to initialize auditor session.\n"); @@ -388,7 +389,7 @@ setup_sessions_and_run (Analysis ana, /** - * Test if the given @a mpub matches the #master_pub. + * Test if the given @a mpub matches the #TALER_ARL_master_pub. * If so, set "found" to GNUNET_YES. * * @param cls a `int *` pointing to "found" @@ -404,29 +405,35 @@ test_master_present (void *cls, (void) exchange_url; if (0 == GNUNET_memcmp (mpub, - &master_pub)) + &TALER_ARL_master_pub)) *found = GNUNET_YES; } +/** + * Setup global variables based on configuration. + * + * @param c configuration to use + * @return #GNUNET_OK on success + */ int -setup_globals (const struct GNUNET_CONFIGURATION_Handle *c) +TALER_ARL_init (const struct GNUNET_CONFIGURATION_Handle *c) { int found; struct TALER_AUDITORDB_Session *as; - cfg = c; + TALER_ARL_cfg = c; start_time = GNUNET_TIME_absolute_get (); - if (0 == GNUNET_is_zero (&master_pub)) + if (0 == GNUNET_is_zero (&TALER_ARL_master_pub)) { /* -m option not given, try configuration */ - char *master_public_key_str; + char *TALER_ARL_master_public_key_str; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, + GNUNET_CONFIGURATION_get_value_string (TALER_ARL_cfg, "exchange", "MASTER_PUBLIC_KEY", - &master_public_key_str)) + &TALER_ARL_master_public_key_str)) { fprintf (stderr, "Pass option -m or set it in the configuration!\n"); @@ -436,35 +443,37 @@ setup_globals (const struct GNUNET_CONFIGURATION_Handle *c) return GNUNET_SYSERR; } if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_public_key_from_string (master_public_key_str, - strlen ( - master_public_key_str), - &master_pub.eddsa_pub)) + GNUNET_CRYPTO_eddsa_public_key_from_string ( + TALER_ARL_master_public_key_str, + strlen ( + TALER_ARL_master_public_key_str), + &TALER_ARL_master_pub. + eddsa_pub)) { fprintf (stderr, "Invalid master public key given in configuration file."); - GNUNET_free (master_public_key_str); + GNUNET_free (TALER_ARL_master_public_key_str); return GNUNET_SYSERR; } - GNUNET_free (master_public_key_str); + GNUNET_free (TALER_ARL_master_public_key_str); } /* end of -m not given */ GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Taler auditor running for exchange master public key %s\n", - TALER_B2S (&master_pub)); + TALER_B2S (&TALER_ARL_master_pub)); if (GNUNET_OK != - TALER_config_get_currency (cfg, - ¤cy)) + TALER_config_get_currency (TALER_ARL_cfg, + &TALER_ARL_currency)) { return GNUNET_SYSERR; } { if (GNUNET_OK != - TALER_config_get_amount (cfg, + TALER_config_get_amount (TALER_ARL_cfg, "taler", "CURRENCY_ROUND_UNIT", - ¤cy_round_unit)) + &TALER_ARL_currency_round_unit)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Invalid or missing amount in `TALER' under `CURRENCY_ROUND_UNIT'\n"); @@ -472,78 +481,86 @@ setup_globals (const struct GNUNET_CONFIGURATION_Handle *c) } } if (NULL == - (edb = TALER_EXCHANGEDB_plugin_load (cfg))) + (TALER_ARL_edb = TALER_EXCHANGEDB_plugin_load (TALER_ARL_cfg))) { fprintf (stderr, "Failed to initialize exchange database plugin.\n"); return GNUNET_SYSERR; } if (NULL == - (adb = TALER_AUDITORDB_plugin_load (cfg))) + (TALER_ARL_adb = TALER_AUDITORDB_plugin_load (TALER_ARL_cfg))) { fprintf (stderr, "Failed to initialize auditor database plugin.\n"); - TALER_EXCHANGEDB_plugin_unload (edb); + TALER_EXCHANGEDB_plugin_unload (TALER_ARL_edb); return GNUNET_SYSERR; } found = GNUNET_NO; - as = adb->get_session (adb->cls); + as = TALER_ARL_adb->get_session (TALER_ARL_adb->cls); if (NULL == as) { fprintf (stderr, "Failed to start session with auditor database.\n"); - TALER_AUDITORDB_plugin_unload (adb); - TALER_EXCHANGEDB_plugin_unload (edb); + TALER_AUDITORDB_plugin_unload (TALER_ARL_adb); + TALER_EXCHANGEDB_plugin_unload (TALER_ARL_edb); return GNUNET_SYSERR; } - (void) adb->list_exchanges (adb->cls, - as, - &test_master_present, - &found); + (void) TALER_ARL_adb->list_exchanges (TALER_ARL_adb->cls, + as, + &test_master_present, + &found); if (GNUNET_NO == found) { fprintf (stderr, "Exchange's master public key `%s' not known to auditor DB. Did you forget to run `taler-auditor-exchange`?\n", - GNUNET_p2s (&master_pub.eddsa_pub)); - TALER_AUDITORDB_plugin_unload (adb); - TALER_EXCHANGEDB_plugin_unload (edb); + GNUNET_p2s (&TALER_ARL_master_pub.eddsa_pub)); + TALER_AUDITORDB_plugin_unload (TALER_ARL_adb); + TALER_EXCHANGEDB_plugin_unload (TALER_ARL_edb); return GNUNET_SYSERR; } - if (restart) + if (TALER_ARL_restart) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Full audit restart requested, dropping old audit data.\n"); + "Full audit TALER_ARL_restart requested, dropping old audit data.\n"); GNUNET_break (GNUNET_OK == - adb->drop_tables (adb->cls, - GNUNET_NO)); - TALER_AUDITORDB_plugin_unload (adb); + TALER_ARL_adb->drop_tables (TALER_ARL_adb->cls, + GNUNET_NO)); + TALER_AUDITORDB_plugin_unload (TALER_ARL_adb); if (NULL == - (adb = TALER_AUDITORDB_plugin_load (cfg))) + (TALER_ARL_adb = TALER_AUDITORDB_plugin_load (TALER_ARL_cfg))) { fprintf (stderr, "Failed to initialize auditor database plugin after drop.\n"); - TALER_EXCHANGEDB_plugin_unload (edb); + TALER_EXCHANGEDB_plugin_unload (TALER_ARL_edb); return GNUNET_SYSERR; } GNUNET_break (GNUNET_OK == - adb->create_tables (adb->cls)); + TALER_ARL_adb->create_tables (TALER_ARL_adb->cls)); } return GNUNET_OK; } +/** + * Generate the report and close connectios to the database. + * + * @param report the report to output, may be NULL for no report + */ void -finish_report (json_t *report) +TALER_ARL_done (json_t *report) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Audit complete\n"); - TALER_AUDITORDB_plugin_unload (adb); - adb = NULL; - TALER_EXCHANGEDB_plugin_unload (edb); - edb = NULL; - json_dumpf (report, - stdout, - JSON_INDENT (2)); - json_decref (report); + TALER_AUDITORDB_plugin_unload (TALER_ARL_adb); + TALER_ARL_adb = NULL; + TALER_EXCHANGEDB_plugin_unload (TALER_ARL_edb); + TALER_ARL_edb = NULL; + if (NULL != report) + { + json_dumpf (report, + stdout, + JSON_INDENT (2)); + json_decref (report); + } } diff --git a/src/auditor/report-lib.h b/src/auditor/report-lib.h index 334ac198e..59229ad14 100644 --- a/src/auditor/report-lib.h +++ b/src/auditor/report-lib.h @@ -30,49 +30,49 @@ /** - * Command-line option "-r": restart audit from scratch + * Command-line option "-r": TALER_ARL_restart audit from scratch */ -extern int restart; +extern int TALER_ARL_restart; /** * Handle to access the exchange's database. */ -extern struct TALER_EXCHANGEDB_Plugin *edb; +extern struct TALER_EXCHANGEDB_Plugin *TALER_ARL_edb; /** - * Which currency are we doing the audit for? + * Which TALER_ARL_currency are we doing the audit for? */ -extern char *currency; +extern char *TALER_ARL_currency; /** - * How many fractional digits does the currency use? + * How many fractional digits does the TALER_ARL_currency use? */ -extern struct TALER_Amount currency_round_unit; +extern struct TALER_Amount TALER_ARL_currency_round_unit; /** * Our configuration. */ -extern const struct GNUNET_CONFIGURATION_Handle *cfg; +extern const struct GNUNET_CONFIGURATION_Handle *TALER_ARL_cfg; /** - * Our session with the #edb. + * Our session with the #TALER_ARL_edb. */ -extern struct TALER_EXCHANGEDB_Session *esession; +extern struct TALER_EXCHANGEDB_Session *TALER_ARL_esession; /** * Handle to access the auditor's database. */ -extern struct TALER_AUDITORDB_Plugin *adb; +extern struct TALER_AUDITORDB_Plugin *TALER_ARL_adb; /** - * Our session with the #adb. + * Our session with the #TALER_ARL_adb. */ -extern struct TALER_AUDITORDB_Session *asession; +extern struct TALER_AUDITORDB_Session *TALER_ARL_asession; /** * Master public key of the exchange to audit. */ -extern struct TALER_MasterPublicKeyP master_pub; +extern struct TALER_MasterPublicKeyP TALER_ARL_master_pub; /** * At what time did the auditor process start? @@ -87,7 +87,7 @@ extern struct GNUNET_TIME_Absolute start_time; * @return human-readable string representing the time */ json_t * -json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at); +TALER_ARL_TALER_ARL_json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at); /** @@ -97,18 +97,18 @@ json_from_time_abs_nbo (struct GNUNET_TIME_AbsoluteNBO at); * @return human-readable string representing the time */ json_t * -json_from_time_abs (struct GNUNET_TIME_Absolute at); +TALER_ARL_json_from_time_abs (struct GNUNET_TIME_Absolute at); /** - * Add @a object to the report @a array. Fail hard if this fails. + * Add @a object to the TALER_ARL_report @a array. Fail hard if this fails. * - * @param array report array to append @a object to + * @param array TALER_ARL_report array to append @a object to * @param object object to append, should be check that it is not NULL */ void -report (json_t *array, - json_t *object); +TALER_ARL_report (json_t *array, + json_t *object); /** @@ -117,10 +117,10 @@ report (json_t *array, * @param dh hash of the denomination public key to look up * @param[out] issue set to detailed information about @a denom_pub, NULL if not found, must * NOT be freed by caller - * @return transaction status code + * @return TALER_ARL_transaction status code */ enum GNUNET_DB_QueryStatus -get_denomination_info_by_hash ( +TALER_ARL_get_denomination_info_by_hash ( const struct GNUNET_HashCode *dh, const struct TALER_DenominationKeyValidityPS **issue); @@ -132,14 +132,15 @@ get_denomination_info_by_hash ( * @param[out] issue set to detailed information about @a denom_pub, NULL if not found, must * NOT be freed by caller * @param[out] dh set to the hash of @a denom_pub, may be NULL - * @return transaction status code + * @return TALER_ARL_transaction status code */ enum GNUNET_DB_QueryStatus -get_denomination_info ( +TALER_ARL_get_denomination_info ( const struct TALER_DenominationPublicKey *denom_pub, const struct TALER_DenominationKeyValidityPS **issue, struct GNUNET_HashCode *dh); + /** * Type of an analysis function. Each analysis function runs in * its own transaction scope and must thus be internally consistent. @@ -148,22 +149,7 @@ get_denomination_info ( * @return transaction status code */ typedef enum GNUNET_DB_QueryStatus -(*Analysis)(void *cls); - - -/** - * Perform the given @a analysis within a transaction scope. - * Commit on success. - * - * @param analysis analysis to run - * @param analysis_cls closure for @a analysis - * @return #GNUNET_OK if @a analysis succeessfully committed, - * #GNUNET_NO if we had an error on commit (retry may help) - * #GNUNET_SYSERR on hard errors - */ -int -transact (Analysis analysis, - void *analysis_cls); +(*TALER_ARL_Analysis)(void *cls); /** @@ -174,15 +160,26 @@ transact (Analysis analysis, * @return #GNUNET_OK on success */ int -setup_sessions_and_run (Analysis ana, - void *ana_cls); +TALER_ARL_setup_sessions_and_run (TALER_ARL_Analysis ana, + void *ana_cls); +/** + * Setup global variables based on configuration. + * + * @param c configuration to use + * @return #GNUNET_OK on success + */ int -setup_globals (const struct GNUNET_CONFIGURATION_Handle *c); +TALER_ARL_init (const struct GNUNET_CONFIGURATION_Handle *c); +/** + * Generate the report and close connectios to the database. + * + * @param report the report to output, may be NULL for no report + */ void -finish_report (json_t *report); +TALER_ARL_done (json_t *report); #endif diff --git a/src/auditor/taler-auditor-aggregation.c b/src/auditor/taler-auditor-aggregation.c deleted file mode 100644 index de249ed52..000000000 --- a/src/auditor/taler-auditor-aggregation.c +++ /dev/null @@ -1,1511 +0,0 @@ -/* - This file is part of TALER - Copyright (C) 2016-2020 Taler Systems SA - - TALER is free software; you can redistribute it and/or modify it under the - terms of the GNU Affero Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - TALER is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. - - You should have received a copy of the GNU Affero Public License along with - TALER; see the file COPYING. If not, see -*/ -/** - * @file auditor/taler-auditor-aggregation.c - * @brief audits an exchange's aggregations. - * @author Christian Grothoff - */ -#include "platform.h" -#include -#include "taler_auditordb_plugin.h" -#include "taler_exchangedb_lib.h" -#include "taler_json_lib.h" -#include "taler_bank_service.h" -#include "taler_signatures.h" -#include "report-lib.h" - - -/** - * Return value from main(). - */ -static int global_ret; - -/** - * Checkpointing our progress for aggregations. - */ -static struct TALER_AUDITORDB_ProgressPointAggregation ppa; - -/** - * Checkpointing our progress for aggregations. - */ -static struct TALER_AUDITORDB_ProgressPointAggregation ppa_start; - -/** - * Array of reports about row inconsitencies. - */ -static json_t *report_row_inconsistencies; - -/** - * Array of reports about irregular wire out entries. - */ -static json_t *report_wire_out_inconsistencies; - -/** - * Total delta between calculated and stored wire out transfers, - * for positive deltas. - */ -static struct TALER_Amount total_wire_out_delta_plus; - -/** - * Total delta between calculated and stored wire out transfers - * for negative deltas. - */ -static struct TALER_Amount total_wire_out_delta_minus; - -/** - * Array of reports about inconsistencies about coins. - */ -static json_t *report_coin_inconsistencies; - -/** - * Profits the exchange made by bad amount calculations on coins. - */ -static struct TALER_Amount total_coin_delta_plus; - -/** - * Losses the exchange made by bad amount calculations on coins. - */ -static struct TALER_Amount total_coin_delta_minus; - -/** - * Report about amount calculation differences (causing profit - * or loss at the exchange). - */ -static json_t *report_amount_arithmetic_inconsistencies; - -/** - * Array of reports about wire fees being ambiguous in terms of validity periods. - */ -static json_t *report_fee_time_inconsistencies; - -/** - * Profits the exchange made by bad amount calculations. - */ -static struct TALER_Amount total_arithmetic_delta_plus; - -/** - * Losses the exchange made by bad amount calculations. - */ -static struct TALER_Amount total_arithmetic_delta_minus; - -/** - * Total aggregation fees earned. - */ -static struct TALER_Amount total_aggregation_fee_income; - -/** - * Array of reports about coin operations with bad signatures. - */ -static json_t *report_bad_sig_losses; - -/** - * Total amount lost by operations for which signatures were invalid. - */ -static struct TALER_Amount total_bad_sig_loss; - - -/** - * Report a (serious) inconsistency in the exchange's database with - * respect to calculations involving amounts. - * - * @param operation what operation had the inconsistency - * @param rowid affected row, UINT64_MAX if row is missing - * @param exchange amount calculated by exchange - * @param auditor amount calculated by auditor - * @param profitable 1 if @a exchange being larger than @a auditor is - * profitable for the exchange for this operation, - * -1 if @a exchange being smaller than @a auditor is - * profitable for the exchange, and 0 if it is unclear - */ -static void -report_amount_arithmetic_inconsistency (const char *operation, - uint64_t rowid, - const struct TALER_Amount *exchange, - const struct TALER_Amount *auditor, - int profitable) -{ - struct TALER_Amount delta; - struct TALER_Amount *target; - - if (0 < TALER_amount_cmp (exchange, - auditor)) - { - /* exchange > auditor */ - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - exchange, - auditor)); - } - else - { - /* auditor < exchange */ - profitable = -profitable; - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - auditor, - exchange)); - } - report (report_amount_arithmetic_inconsistencies, - json_pack ("{s:s, s:I, s:o, s:o, s:I}", - "operation", operation, - "rowid", (json_int_t) rowid, - "exchange", TALER_JSON_from_amount (exchange), - "auditor", TALER_JSON_from_amount (auditor), - "profitable", (json_int_t) profitable)); - if (0 != profitable) - { - target = (1 == profitable) - ? &total_arithmetic_delta_plus - : &total_arithmetic_delta_minus; - GNUNET_break (GNUNET_OK == - TALER_amount_add (target, - target, - &delta)); - } -} - - -/** - * Report a (serious) inconsistency in the exchange's database with - * respect to calculations involving amounts of a coin. - * - * @param operation what operation had the inconsistency - * @param coin_pub affected coin - * @param exchange amount calculated by exchange - * @param auditor amount calculated by auditor - * @param profitable 1 if @a exchange being larger than @a auditor is - * profitable for the exchange for this operation, - * -1 if @a exchange being smaller than @a auditor is - * profitable for the exchange, and 0 if it is unclear - */ -static void -report_coin_arithmetic_inconsistency (const char *operation, - const struct - TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_Amount *exchange, - const struct TALER_Amount *auditor, - int profitable) -{ - struct TALER_Amount delta; - struct TALER_Amount *target; - - if (0 < TALER_amount_cmp (exchange, - auditor)) - { - /* exchange > auditor */ - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - exchange, - auditor)); - } - else - { - /* auditor < exchange */ - profitable = -profitable; - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - auditor, - exchange)); - } - report (report_coin_inconsistencies, - json_pack ("{s:s, s:o, s:o, s:o, s:I}", - "operation", operation, - "coin_pub", GNUNET_JSON_from_data_auto (coin_pub), - "exchange", TALER_JSON_from_amount (exchange), - "auditor", TALER_JSON_from_amount (auditor), - "profitable", (json_int_t) profitable)); - if (0 != profitable) - { - target = (1 == profitable) - ? &total_coin_delta_plus - : &total_coin_delta_minus; - GNUNET_break (GNUNET_OK == - TALER_amount_add (target, - target, - &delta)); - } -} - - -/** - * Report a (serious) inconsistency in the exchange's database. - * - * @param table affected table - * @param rowid affected row, UINT64_MAX if row is missing - * @param diagnostic message explaining the problem - */ -static void -report_row_inconsistency (const char *table, - uint64_t rowid, - const char *diagnostic) -{ - report (report_row_inconsistencies, - json_pack ("{s:s, s:I, s:s}", - "table", table, - "row", (json_int_t) rowid, - "diagnostic", diagnostic)); -} - - -/* *********************** Analyze aggregations ******************** */ -/* This logic checks that the aggregator did the right thing - paying each merchant what they were due (and on time). */ - - -/** - * Information about wire fees charged by the exchange. - */ -struct WireFeeInfo -{ - - /** - * Kept in a DLL. - */ - struct WireFeeInfo *next; - - /** - * Kept in a DLL. - */ - struct WireFeeInfo *prev; - - /** - * When does the fee go into effect (inclusive). - */ - struct GNUNET_TIME_Absolute start_date; - - /** - * When does the fee stop being in effect (exclusive). - */ - struct GNUNET_TIME_Absolute end_date; - - /** - * How high is the wire fee. - */ - struct TALER_Amount wire_fee; - - /** - * How high is the closing fee. - */ - struct TALER_Amount closing_fee; - -}; - - -/** - * Closure for callbacks during #analyze_merchants(). - */ -struct AggregationContext -{ - - /** - * DLL of wire fees charged by the exchange. - */ - struct WireFeeInfo *fee_head; - - /** - * DLL of wire fees charged by the exchange. - */ - struct WireFeeInfo *fee_tail; - - /** - * Final result status. - */ - enum GNUNET_DB_QueryStatus qs; -}; - - -/** - * Closure for #wire_transfer_information_cb. - */ -struct WireCheckContext -{ - - /** - * Corresponding merchant context. - */ - struct AggregationContext *ac; - - /** - * Total deposits claimed by all transactions that were aggregated - * under the given @e wtid. - */ - struct TALER_Amount total_deposits; - - /** - * Hash of the wire transfer details of the receiver. - */ - struct GNUNET_HashCode h_wire; - - /** - * Execution time of the wire transfer. - */ - struct GNUNET_TIME_Absolute date; - - /** - * Database transaction status. - */ - enum GNUNET_DB_QueryStatus qs; - -}; - - -/** - * Check coin's transaction history for plausibility. Does NOT check - * the signatures (those are checked independently), but does calculate - * the amounts for the aggregation table and checks that the total - * claimed coin value is within the value of the coin's denomination. - * - * @param coin_pub public key of the coin (for reporting) - * @param h_contract_terms hash of the proposal for which we calculate the amount - * @param merchant_pub public key of the merchant (who is allowed to issue refunds) - * @param issue denomination information about the coin - * @param tl_head head of transaction history to verify - * @param[out] merchant_gain amount the coin contributes to the wire transfer to the merchant - * @param[out] deposit_gain amount the coin contributes excluding refunds - * @return #GNUNET_OK on success, #GNUNET_SYSERR on error - */ -static int -check_transaction_history_for_deposit (const struct - TALER_CoinSpendPublicKeyP *coin_pub, - const struct - GNUNET_HashCode *h_contract_terms, - const struct - TALER_MerchantPublicKeyP *merchant_pub, - const struct - TALER_DenominationKeyValidityPS *issue, - const struct - TALER_EXCHANGEDB_TransactionList *tl_head, - struct TALER_Amount *merchant_gain, - struct TALER_Amount *deposit_gain) -{ - struct TALER_Amount expenditures; - struct TALER_Amount refunds; - struct TALER_Amount spent; - struct TALER_Amount value; - struct TALER_Amount merchant_loss; - struct TALER_Amount final_gain; - const struct TALER_Amount *deposit_fee; - int refund_deposit_fee; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Checking transaction history of coin %s\n", - TALER_B2S (coin_pub)); - - GNUNET_assert (NULL != tl_head); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &expenditures)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &refunds)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - merchant_gain)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &merchant_loss)); - /* Go over transaction history to compute totals; note that we do not - know the order, so instead of subtracting we compute positive - (deposit, melt) and negative (refund) values separately here, - and then subtract the negative from the positive after the loop. */ - refund_deposit_fee = GNUNET_NO; - deposit_fee = NULL; - for (const struct TALER_EXCHANGEDB_TransactionList *tl = tl_head; - NULL != tl; - tl = tl->next) - { - const struct TALER_Amount *amount_with_fee; - const struct TALER_Amount *fee; - const struct TALER_AmountNBO *fee_dki; - struct TALER_Amount tmp; - - switch (tl->type) - { - case TALER_EXCHANGEDB_TT_DEPOSIT: - /* check wire and h_wire are consistent */ - { - struct GNUNET_HashCode hw; - - if (GNUNET_OK != - TALER_JSON_merchant_wire_signature_hash ( - tl->details.deposit->receiver_wire_account, - &hw)) - { - report_row_inconsistency ("deposits", - tl->serial_id, - "wire value malformed"); - } - else if (0 != - GNUNET_memcmp (&hw, - &tl->details.deposit->h_wire)) - { - report_row_inconsistency ("deposits", - tl->serial_id, - "h(wire) does not match wire"); - } - } - amount_with_fee = &tl->details.deposit->amount_with_fee; - fee = &tl->details.deposit->deposit_fee; - fee_dki = &issue->fee_deposit; - if (GNUNET_OK != - TALER_amount_add (&expenditures, - &expenditures, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - /* Check if this deposit is within the remit of the aggregation - we are investigating, if so, include it in the totals. */ - if ( (0 == GNUNET_memcmp (merchant_pub, - &tl->details.deposit->merchant_pub)) && - (0 == GNUNET_memcmp (h_contract_terms, - &tl->details.deposit->h_contract_terms)) ) - { - struct TALER_Amount amount_without_fee; - - if (GNUNET_OK != - TALER_amount_subtract (&amount_without_fee, - amount_with_fee, - fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (merchant_gain, - merchant_gain, - &amount_without_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Detected applicable deposit of %s\n", - TALER_amount2s (&amount_without_fee)); - deposit_fee = fee; - } - /* Check that the fees given in the transaction list and in dki match */ - TALER_amount_ntoh (&tmp, - fee_dki); - if (0 != - TALER_amount_cmp (&tmp, - fee)) - { - /* Disagreement in fee structure within DB, should be impossible! */ - GNUNET_break (0); - return GNUNET_SYSERR; - } - break; - case TALER_EXCHANGEDB_TT_MELT: - amount_with_fee = &tl->details.melt->amount_with_fee; - fee = &tl->details.melt->melt_fee; - fee_dki = &issue->fee_refresh; - if (GNUNET_OK != - TALER_amount_add (&expenditures, - &expenditures, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - /* Check that the fees given in the transaction list and in dki match */ - TALER_amount_ntoh (&tmp, - fee_dki); - if (0 != - TALER_amount_cmp (&tmp, - fee)) - { - /* Disagreement in fee structure within DB, should be impossible! */ - GNUNET_break (0); - return GNUNET_SYSERR; - } - break; - case TALER_EXCHANGEDB_TT_REFUND: - amount_with_fee = &tl->details.refund->refund_amount; - fee = &tl->details.refund->refund_fee; - fee_dki = &issue->fee_refund; - if (GNUNET_OK != - TALER_amount_add (&refunds, - &refunds, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&expenditures, - &expenditures, - fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - /* Check if this refund is within the remit of the aggregation - we are investigating, if so, include it in the totals. */ - if ( (0 == GNUNET_memcmp (merchant_pub, - &tl->details.refund->merchant_pub)) && - (0 == GNUNET_memcmp (h_contract_terms, - &tl->details.refund->h_contract_terms)) ) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Detected applicable refund of %s\n", - TALER_amount2s (amount_with_fee)); - if (GNUNET_OK != - TALER_amount_add (&merchant_loss, - &merchant_loss, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - refund_deposit_fee = GNUNET_YES; - } - /* Check that the fees given in the transaction list and in dki match */ - TALER_amount_ntoh (&tmp, - fee_dki); - if (0 != - TALER_amount_cmp (&tmp, - fee)) - { - /* Disagreement in fee structure within DB, should be impossible! */ - GNUNET_break (0); - return GNUNET_SYSERR; - } - break; - case TALER_EXCHANGEDB_TT_OLD_COIN_RECOUP: - amount_with_fee = &tl->details.old_coin_recoup->value; - if (GNUNET_OK != - TALER_amount_add (&refunds, - &refunds, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - break; - case TALER_EXCHANGEDB_TT_RECOUP: - amount_with_fee = &tl->details.recoup->value; - if (GNUNET_OK != - TALER_amount_add (&expenditures, - &expenditures, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - break; - case TALER_EXCHANGEDB_TT_RECOUP_REFRESH: - amount_with_fee = &tl->details.recoup_refresh->value; - if (GNUNET_OK != - TALER_amount_add (&expenditures, - &expenditures, - amount_with_fee)) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - break; - } - } /* for 'tl' */ - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Deposits without fees are %s\n", - TALER_amount2s (merchant_gain)); - - /* Calculate total balance change, i.e. expenditures (recoup, deposit, refresh) - minus refunds (refunds, recoup-to-old) */ - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Subtracting refunds of %s from coin value loss\n", - TALER_amount2s (&refunds)); - if (GNUNET_SYSERR == - TALER_amount_subtract (&spent, - &expenditures, - &refunds)) - { - /* refunds above expenditures? Bad! */ - report_coin_arithmetic_inconsistency ("refund (balance)", - coin_pub, - &expenditures, - &refunds, - 1); - return GNUNET_SYSERR; - } - - /* Now check that 'spent' is less or equal than the total coin value */ - TALER_amount_ntoh (&value, - &issue->value); - if (1 == TALER_amount_cmp (&spent, - &value)) - { - /* spent > value */ - report_coin_arithmetic_inconsistency ("spend", - coin_pub, - &spent, - &value, - -1); - return GNUNET_SYSERR; - } - - /* Finally, update @a merchant_gain by subtracting what he "lost" - from refunds */ - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Merchant 'loss' due to refunds is %s\n", - TALER_amount2s (&merchant_loss)); - *deposit_gain = *merchant_gain; - if ( (GNUNET_YES == refund_deposit_fee) && - (NULL != deposit_fee) ) - { - /* We had a /deposit operation AND a /refund operation, - and should thus not charge the merchant the /deposit fee */ - GNUNET_assert (GNUNET_OK == - TALER_amount_add (merchant_gain, - merchant_gain, - deposit_fee)); - } - if (GNUNET_SYSERR == - TALER_amount_subtract (&final_gain, - merchant_gain, - &merchant_loss)) - { - /* refunds above deposits? Bad! */ - report_coin_arithmetic_inconsistency ("refund (merchant)", - coin_pub, - merchant_gain, - &merchant_loss, - 1); - return GNUNET_SYSERR; - } - *merchant_gain = final_gain; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Final merchant gain after refunds is %s\n", - TALER_amount2s (deposit_gain)); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Coin %s contributes %s to contract %s\n", - TALER_B2S (coin_pub), - TALER_amount2s (merchant_gain), - GNUNET_h2s (h_contract_terms)); - return GNUNET_OK; -} - - -/** - * Function called with the results of the lookup of the - * transaction data associated with a wire transfer identifier. - * - * @param cls a `struct WireCheckContext` - * @param rowid which row in the table is the information from (for diagnostics) - * @param merchant_pub public key of the merchant (should be same for all callbacks with the same @e cls) - * @param h_wire hash of wire transfer details of the merchant (should be same for all callbacks with the same @e cls) - * @param account_details where did we transfer the funds? - * @param exec_time execution time of the wire transfer (should be same for all callbacks with the same @e cls) - * @param h_contract_terms which proposal was this payment about - * @param denom_pub denomination of @a coin_pub - * @param coin_pub which public key was this payment about - * @param coin_value amount contributed by this coin in total (with fee), - * but excluding refunds by this coin - * @param deposit_fee applicable deposit fee for this coin, actual - * fees charged may differ if coin was refunded - */ -static void -wire_transfer_information_cb ( - void *cls, - uint64_t rowid, - const struct TALER_MerchantPublicKeyP *merchant_pub, - const struct GNUNET_HashCode *h_wire, - const json_t *account_details, - struct GNUNET_TIME_Absolute exec_time, - const struct GNUNET_HashCode *h_contract_terms, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_Amount *coin_value, - const struct TALER_Amount *deposit_fee) -{ - struct WireCheckContext *wcc = cls; - const struct TALER_DenominationKeyValidityPS *issue; - struct TALER_Amount computed_value; - struct TALER_Amount coin_value_without_fee; - struct TALER_Amount total_deposit_without_refunds; - struct TALER_EXCHANGEDB_TransactionList *tl; - struct TALER_CoinPublicInfo coin; - enum GNUNET_DB_QueryStatus qs; - struct GNUNET_HashCode hw; - - if (GNUNET_OK != - TALER_JSON_merchant_wire_signature_hash (account_details, - &hw)) - { - report_row_inconsistency ("aggregation", - rowid, - "failed to compute hash of given wire data"); - } - else if (0 != - GNUNET_memcmp (&hw, - h_wire)) - { - report_row_inconsistency ("aggregation", - rowid, - "database contains wrong hash code for wire details"); - } - - /* Obtain coin's transaction history */ - qs = edb->get_coin_transactions (edb->cls, - esession, - coin_pub, - GNUNET_YES, - &tl); - if ( (qs < 0) || - (NULL == tl) ) - { - wcc->qs = qs; - report_row_inconsistency ("aggregation", - rowid, - "no transaction history for coin claimed in aggregation"); - return; - } - qs = edb->get_known_coin (edb->cls, - esession, - coin_pub, - &coin); - if (qs < 0) - { - GNUNET_break (0); /* this should be a foreign key violation at this point! */ - wcc->qs = qs; - report_row_inconsistency ("aggregation", - rowid, - "could not get coin details for coin claimed in aggregation"); - return; - } - - qs = get_denomination_info_by_hash (&coin.denom_pub_hash, - &issue); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_CRYPTO_rsa_signature_free (coin.denom_sig.rsa_signature); - edb->free_coin_transaction_list (edb->cls, - tl); - if (0 == qs) - report_row_inconsistency ("aggregation", - rowid, - "could not find denomination key for coin claimed in aggregation"); - else - wcc->qs = qs; - return; - } - if (GNUNET_OK != - TALER_test_coin_valid (&coin, - denom_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "wire", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (coin_value), - "key_pub", GNUNET_JSON_from_data_auto ( - &issue->denom_hash))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - coin_value)); - GNUNET_CRYPTO_rsa_signature_free (coin.denom_sig.rsa_signature); - edb->free_coin_transaction_list (edb->cls, - tl); - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - report_row_inconsistency ("deposit", - rowid, - "coin denomination signature invalid"); - return; - } - GNUNET_CRYPTO_rsa_signature_free (coin.denom_sig.rsa_signature); - coin.denom_sig.rsa_signature = NULL; /* just to be sure */ - GNUNET_assert (NULL != issue); /* mostly to help static analysis */ - /* Check transaction history to see if it supports aggregate - valuation */ - if (GNUNET_OK != - check_transaction_history_for_deposit (coin_pub, - h_contract_terms, - merchant_pub, - issue, - tl, - &computed_value, - &total_deposit_without_refunds)) - { - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - report_row_inconsistency ("coin history", - rowid, - "failed to verify coin history (for deposit)"); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Coin contributes %s to aggregate (deposits after fees and refunds)\n", - TALER_amount2s (&computed_value)); - if (GNUNET_SYSERR == - TALER_amount_subtract (&coin_value_without_fee, - coin_value, - deposit_fee)) - { - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - report_amount_arithmetic_inconsistency ("aggregation (fee structure)", - rowid, - coin_value, - deposit_fee, - -1); - return; - } - if (0 != - TALER_amount_cmp (&total_deposit_without_refunds, - &coin_value_without_fee)) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Expected coin contribution of %s to aggregate\n", - TALER_amount2s (&coin_value_without_fee)); - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - report_amount_arithmetic_inconsistency ("aggregation (contribution)", - rowid, - &coin_value_without_fee, - &total_deposit_without_refunds, - -1); - } - edb->free_coin_transaction_list (edb->cls, - tl); - - /* Check other details of wire transfer match */ - if (0 != GNUNET_memcmp (h_wire, - &wcc->h_wire)) - { - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - report_row_inconsistency ("aggregation", - rowid, - "target of outgoing wire transfer do not match hash of wire from deposit"); - } - if (exec_time.abs_value_us != wcc->date.abs_value_us) - { - /* This should be impossible from database constraints */ - GNUNET_break (0); - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - report_row_inconsistency ("aggregation", - rowid, - "date given in aggregate does not match wire transfer date"); - } - - /* Add coin's contribution to total aggregate value */ - { - struct TALER_Amount res; - - if (GNUNET_OK != - TALER_amount_add (&res, - &wcc->total_deposits, - &computed_value)) - { - GNUNET_break (0); - wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return; - } - wcc->total_deposits = res; - } -} - - -/** - * Lookup the wire fee that the exchange charges at @a timestamp. - * - * @param ac context for caching the result - * @param method method of the wire plugin - * @param timestamp time for which we need the fee - * @return NULL on error (fee unknown) - */ -static const struct TALER_Amount * -get_wire_fee (struct AggregationContext *ac, - const char *method, - struct GNUNET_TIME_Absolute timestamp) -{ - struct WireFeeInfo *wfi; - struct WireFeeInfo *pos; - struct TALER_MasterSignatureP master_sig; - - /* Check if fee is already loaded in cache */ - for (pos = ac->fee_head; NULL != pos; pos = pos->next) - { - if ( (pos->start_date.abs_value_us <= timestamp.abs_value_us) && - (pos->end_date.abs_value_us > timestamp.abs_value_us) ) - return &pos->wire_fee; - if (pos->start_date.abs_value_us > timestamp.abs_value_us) - break; - } - - /* Lookup fee in exchange database */ - wfi = GNUNET_new (struct WireFeeInfo); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != - edb->get_wire_fee (edb->cls, - esession, - method, - timestamp, - &wfi->start_date, - &wfi->end_date, - &wfi->wire_fee, - &wfi->closing_fee, - &master_sig)) - { - GNUNET_break (0); - GNUNET_free (wfi); - return NULL; - } - - /* Check signature. (This is not terribly meaningful as the exchange can - easily make this one up, but it means that we have proof that the master - key was used for inconsistent wire fees if a merchant complains.) */ - { - struct TALER_MasterWireFeePS wf; - - wf.purpose.purpose = htonl (TALER_SIGNATURE_MASTER_WIRE_FEES); - wf.purpose.size = htonl (sizeof (wf)); - GNUNET_CRYPTO_hash (method, - strlen (method) + 1, - &wf.h_wire_method); - wf.start_date = GNUNET_TIME_absolute_hton (wfi->start_date); - wf.end_date = GNUNET_TIME_absolute_hton (wfi->end_date); - TALER_amount_hton (&wf.wire_fee, - &wfi->wire_fee); - TALER_amount_hton (&wf.closing_fee, - &wfi->closing_fee); - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_MASTER_WIRE_FEES, - &wf.purpose, - &master_sig.eddsa_signature, - &master_pub.eddsa_pub)) - { - report_row_inconsistency ("wire-fee", - timestamp.abs_value_us, - "wire fee signature invalid at given time"); - } - } - - /* Established fee, keep in sorted list */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Wire fee is %s starting at %s\n", - TALER_amount2s (&wfi->wire_fee), - GNUNET_STRINGS_absolute_time_to_string (wfi->start_date)); - if ( (NULL == pos) || - (NULL == pos->prev) ) - GNUNET_CONTAINER_DLL_insert (ac->fee_head, - ac->fee_tail, - wfi); - else - GNUNET_CONTAINER_DLL_insert_after (ac->fee_head, - ac->fee_tail, - pos->prev, - wfi); - /* Check non-overlaping fee invariant */ - if ( (NULL != wfi->prev) && - (wfi->prev->end_date.abs_value_us > wfi->start_date.abs_value_us) ) - { - report (report_fee_time_inconsistencies, - json_pack ("{s:s, s:s, s:o}", - "type", method, - "diagnostic", "start date before previous end date", - "time", json_from_time_abs (wfi->start_date))); - } - if ( (NULL != wfi->next) && - (wfi->next->start_date.abs_value_us >= wfi->end_date.abs_value_us) ) - { - report (report_fee_time_inconsistencies, - json_pack ("{s:s, s:s, s:o}", - "type", method, - "diagnostic", "end date date after next start date", - "time", json_from_time_abs (wfi->end_date))); - } - return &wfi->wire_fee; -} - - -/** - * Check that a wire transfer made by the exchange is valid - * (has matching deposits). - * - * @param cls a `struct AggregationContext` - * @param rowid identifier of the respective row in the database - * @param date timestamp of the wire transfer (roughly) - * @param wtid wire transfer subject - * @param wire wire transfer details of the receiver - * @param amount amount that was wired - * @return #GNUNET_OK to continue, #GNUNET_SYSERR to stop iteration - */ -static int -check_wire_out_cb (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute date, - const struct TALER_WireTransferIdentifierRawP *wtid, - const json_t *wire, - const struct TALER_Amount *amount) -{ - struct AggregationContext *ac = cls; - struct WireCheckContext wcc; - struct TALER_Amount final_amount; - struct TALER_Amount exchange_gain; - enum GNUNET_DB_QueryStatus qs; - char *method; - - /* should be monotonically increasing */ - GNUNET_assert (rowid >= ppa.last_wire_out_serial_id); - ppa.last_wire_out_serial_id = rowid + 1; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Checking wire transfer %s over %s performed on %s\n", - TALER_B2S (wtid), - TALER_amount2s (amount), - GNUNET_STRINGS_absolute_time_to_string (date)); - if (NULL == (method = TALER_JSON_wire_to_method (wire))) - { - report_row_inconsistency ("wire_out", - rowid, - "specified wire address lacks method"); - return GNUNET_OK; - } - - wcc.ac = ac; - wcc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - wcc.date = date; - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount->currency, - &wcc.total_deposits)); - if (GNUNET_OK != - TALER_JSON_merchant_wire_signature_hash (wire, - &wcc.h_wire)) - { - GNUNET_break (0); - GNUNET_free (method); - return GNUNET_SYSERR; - } - qs = edb->lookup_wire_transfer (edb->cls, - esession, - wtid, - &wire_transfer_information_cb, - &wcc); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - ac->qs = qs; - GNUNET_free (method); - return GNUNET_SYSERR; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != wcc.qs) - { - /* Note: detailed information was already logged - in #wire_transfer_information_cb, so here we - only log for debugging */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Inconsitency for wire_out %llu (WTID %s) detected\n", - (unsigned long long) rowid, - TALER_B2S (wtid)); - } - - - /* Subtract aggregation fee from total (if possible) */ - { - const struct TALER_Amount *wire_fee; - - wire_fee = get_wire_fee (ac, - method, - date); - if (NULL == wire_fee) - { - report_row_inconsistency ("wire-fee", - date.abs_value_us, - "wire fee unavailable for given time"); - /* If fee is unknown, we just assume the fee is zero */ - final_amount = wcc.total_deposits; - } - else if (GNUNET_SYSERR == - TALER_amount_subtract (&final_amount, - &wcc.total_deposits, - wire_fee)) - { - report_amount_arithmetic_inconsistency ("wire out (fee structure)", - rowid, - &wcc.total_deposits, - wire_fee, - -1); - /* If fee arithmetic fails, we just assume the fee is zero */ - final_amount = wcc.total_deposits; - } - } - GNUNET_free (method); - - /* Round down to amount supported by wire method */ - GNUNET_break (GNUNET_SYSERR != - TALER_amount_round_down (&final_amount, - ¤cy_round_unit)); - - /* Calculate the exchange's gain as the fees plus rounding differences! */ - if (GNUNET_SYSERR == - TALER_amount_subtract (&exchange_gain, - &wcc.total_deposits, - &final_amount)) - { - GNUNET_break (0); - ac->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - - /* Sum up aggregation fees (we simply include the rounding gains) */ - if (GNUNET_OK != - TALER_amount_add (&total_aggregation_fee_income, - &total_aggregation_fee_income, - &exchange_gain)) - { - GNUNET_break (0); - ac->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - - /* Check that calculated amount matches actual amount */ - if (0 != TALER_amount_cmp (amount, - &final_amount)) - { - struct TALER_Amount delta; - - if (0 < TALER_amount_cmp (amount, - &final_amount)) - { - /* amount > final_amount */ - GNUNET_assert (GNUNET_OK == - TALER_amount_subtract (&delta, - amount, - &final_amount)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&total_wire_out_delta_plus, - &total_wire_out_delta_plus, - &delta)); - } - else - { - /* amount < final_amount */ - GNUNET_assert (GNUNET_OK == - TALER_amount_subtract (&delta, - &final_amount, - amount)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&total_wire_out_delta_minus, - &total_wire_out_delta_minus, - &delta)); - } - - report (report_wire_out_inconsistencies, - json_pack ("{s:O, s:I, s:o, s:o}", - "destination_account", wire, - "rowid", (json_int_t) rowid, - "expected", - TALER_JSON_from_amount (&final_amount), - "claimed", - TALER_JSON_from_amount (amount))); - return GNUNET_OK; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Wire transfer %s is OK\n", - TALER_B2S (wtid)); - return GNUNET_OK; -} - - -/** - * Analyze the exchange aggregator's payment processing. - * - * @param cls closure - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -analyze_aggregations (void *cls) -{ - struct AggregationContext ac; - struct WireFeeInfo *wfi; - enum GNUNET_DB_QueryStatus qsx; - enum GNUNET_DB_QueryStatus qs; - enum GNUNET_DB_QueryStatus qsp; - - (void) cls; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Analyzing aggregations\n"); - qsp = adb->get_auditor_progress_aggregation (adb->cls, - asession, - &master_pub, - &ppa); - if (0 > qsp) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); - return qsp; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) - { - GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, - _ ( - "First analysis using this auditor, starting audit from scratch\n")); - } - else - { - ppa_start = ppa; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Resuming aggregation audit at %llu\n"), - (unsigned long long) ppa.last_wire_out_serial_id); - } - - memset (&ac, - 0, - sizeof (ac)); - qsx = adb->get_wire_fee_summary (adb->cls, - asession, - &master_pub, - &total_aggregation_fee_income); - if (0 > qsx) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); - return qsx; - } - ac.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - qs = edb->select_wire_out_above_serial_id (edb->cls, - esession, - ppa.last_wire_out_serial_id, - &check_wire_out_cb, - &ac); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - ac.qs = qs; - } - while (NULL != (wfi = ac.fee_head)) - { - GNUNET_CONTAINER_DLL_remove (ac.fee_head, - ac.fee_tail, - wfi); - GNUNET_free (wfi); - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - /* there were no wire out entries to be looked at, we are done */ - return qs; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ac.qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == ac.qs); - return ac.qs; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsx) - ac.qs = adb->insert_wire_fee_summary (adb->cls, - asession, - &master_pub, - &total_aggregation_fee_income); - else - ac.qs = adb->update_wire_fee_summary (adb->cls, - asession, - &master_pub, - &total_aggregation_fee_income); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ac.qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == ac.qs); - return ac.qs; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) - qs = adb->update_auditor_progress_aggregation (adb->cls, - asession, - &master_pub, - &ppa); - else - qs = adb->insert_auditor_progress_aggregation (adb->cls, - asession, - &master_pub, - &ppa); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to update auditor DB, not recording progress\n"); - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Concluded aggregation audit step at %llu\n"), - (unsigned long long) ppa.last_wire_out_serial_id); - - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; -} - - -/** - * Main function that will be run. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - json_t *report; - - (void) cls; - (void) args; - (void) cfgfile; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Launching auditor\n"); - if (GNUNET_OK != - setup_globals (c)) - { - global_ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting audit\n"); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_aggregation_fee_income)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_wire_out_delta_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_wire_out_delta_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_arithmetic_delta_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_arithmetic_delta_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_coin_delta_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_coin_delta_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_sig_loss)); - GNUNET_assert (NULL != - (report_row_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_wire_out_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_coin_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_amount_arithmetic_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_bad_sig_losses = json_array ())); - GNUNET_assert (NULL != - (report_fee_time_inconsistencies = json_array ())); - setup_sessions_and_run (&analyze_aggregations, - NULL); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Audit complete\n"); - report = json_pack ("{s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:I, s:I," - " s:o, s:o }", - /* blocks #1 */ - "wire_out_inconsistencies", - report_wire_out_inconsistencies, - "total_wire_out_delta_plus", - TALER_JSON_from_amount (&total_wire_out_delta_plus), - "total_wire_out_delta_minus", - TALER_JSON_from_amount (&total_wire_out_delta_minus), - /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ - "bad_sig_losses", - report_bad_sig_losses, - /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ - "total_bad_sig_loss", - TALER_JSON_from_amount (&total_bad_sig_loss), - /* block #2 */ - /* Tested in test-auditor.sh #14/#15 */ - "row_inconsistencies", - report_row_inconsistencies, - "coin_inconsistencies", - report_coin_inconsistencies, - "total_coin_delta_plus", - TALER_JSON_from_amount (&total_coin_delta_plus), - "total_coin_delta_minus", - TALER_JSON_from_amount (&total_coin_delta_minus), - "amount_arithmetic_inconsistencies", - report_amount_arithmetic_inconsistencies, - /* block #3 */ - "total_arithmetic_delta_plus", - TALER_JSON_from_amount (&total_arithmetic_delta_plus), - "total_arithmetic_delta_minus", - TALER_JSON_from_amount (&total_arithmetic_delta_minus), - "total_aggregation_fee_income", - TALER_JSON_from_amount (&total_aggregation_fee_income), - "start_ppa_wire_out_serial_id", - (json_int_t) ppa_start.last_wire_out_serial_id, - "end_ppa_wire_out_serial_id", - (json_int_t) ppa.last_wire_out_serial_id, - /* block #4 */ - "auditor_start_time", json_from_time_abs (start_time), - "auditor_end_time", json_from_time_abs ( - GNUNET_TIME_absolute_get ()) - ); - GNUNET_break (NULL != report); - finish_report (report); -} - - -/** - * The main function of the database initialization tool. - * Used to initialize the Taler Exchange's database. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, - char *const *argv) -{ - const struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_base32_auto ('m', - "exchange-key", - "KEY", - "public key of the exchange (Crockford base32 encoded)", - &master_pub), - GNUNET_GETOPT_option_flag ('r', - "restart", - "restart audit from the beginning (required on first run)", - &restart), - GNUNET_GETOPT_option_timetravel ('T', - "timetravel"), - GNUNET_GETOPT_OPTION_END - }; - - /* force linker to link against libtalerutil; if we do - not do this, the linker may "optimize" libtalerutil - away and skip #TALER_OS_init(), which we do need */ - (void) TALER_project_data_default (); - GNUNET_assert (GNUNET_OK == - GNUNET_log_setup ("taler-auditor-aggregation", - "MESSAGE", - NULL)); - if (GNUNET_OK != - GNUNET_PROGRAM_run (argc, - argv, - "taler-auditor-aggregation", - "Audit Taler exchange aggregation activity", - options, - &run, - NULL)) - return 1; - return global_ret; -} - - -/* end of taler-auditor-aggregation.c */ diff --git a/src/auditor/taler-auditor-coins.c b/src/auditor/taler-auditor-coins.c deleted file mode 100644 index d895c0433..000000000 --- a/src/auditor/taler-auditor-coins.c +++ /dev/null @@ -1,2346 +0,0 @@ -/* - This file is part of TALER - Copyright (C) 2016-2020 Taler Systems SA - - TALER is free software; you can redistribute it and/or modify it under the - terms of the GNU Affero Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - TALER is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. - - You should have received a copy of the GNU Affero Public License along with - TALER; see the file COPYING. If not, see -*/ -/** - * @file auditor/taler-auditor.c - * @brief audits an exchange database. - * @author Christian Grothoff - * - * NOTE: - * - This auditor does not verify that 'reserves_in' actually matches - * the wire transfers from the bank. This needs to be checked separately! - * - Similarly, we do not check that the outgoing wire transfers match those - * given in the 'wire_out' table. This needs to be checked separately! - * - * TODO: - * - reorganize: different passes are combined in one tool and one - * file here, we should split this up! - * - likely should do an iteration over known_coins instead of checking - * those signatures again and again - * - might want to bite the bullet and do asynchronous signature - * verification to improve parallelism / speed -- we'll need to scale - * this eventually anyway! - * - * UNDECIDED: - * - do we care about checking the 'done' flag in deposit_cb? - */ -#include "platform.h" -#include -#include "taler_auditordb_plugin.h" -#include "taler_exchangedb_lib.h" -#include "taler_json_lib.h" -#include "taler_bank_service.h" -#include "taler_signatures.h" -#include "report-lib.h" - -/** - * How many coin histories do we keep in RAM at any given point in - * time? Used bound memory consumption of the auditor. Larger values - * reduce database accesses. - * - * Set to a VERY low value here for testing. Practical values may be - * in the millions. - */ -#define MAX_COIN_SUMMARIES 4 - -/** - * Use a 1 day grace period to deal with clocks not being perfectly synchronized. - */ -#define DEPOSIT_GRACE_PERIOD GNUNET_TIME_UNIT_DAYS - -/** - * Return value from main(). - */ -static int global_ret; - -/** - * Checkpointing our progress for coins. - */ -static struct TALER_AUDITORDB_ProgressPointCoin ppc; - -/** - * Checkpointing our progress for coins. - */ -static struct TALER_AUDITORDB_ProgressPointCoin ppc_start; - -/** - * Array of reports about denomination keys with an - * emergency (more value deposited than withdrawn) - */ -static json_t *report_emergencies; - -/** - * Array of reports about denomination keys with an - * emergency (more coins deposited than withdrawn) - */ -static json_t *report_emergencies_by_count; - -/** - * Array of reports about row inconsitencies. - */ -static json_t *report_row_inconsistencies; - -/** - * Report about amount calculation differences (causing profit - * or loss at the exchange). - */ -static json_t *report_amount_arithmetic_inconsistencies; - -/** - * Profits the exchange made by bad amount calculations. - */ -static struct TALER_Amount total_arithmetic_delta_plus; - -/** - * Losses the exchange made by bad amount calculations. - */ -static struct TALER_Amount total_arithmetic_delta_minus; - -/** - * Total amount reported in all calls to #report_emergency_by_count(). - */ -static struct TALER_Amount reported_emergency_risk_by_count; - -/** - * Total amount reported in all calls to #report_emergency_by_amount(). - */ -static struct TALER_Amount reported_emergency_risk_by_amount; - -/** - * Total amount in losses reported in all calls to #report_emergency_by_amount(). - */ -static struct TALER_Amount reported_emergency_loss; - -/** - * Total amount in losses reported in all calls to #report_emergency_by_count(). - */ -static struct TALER_Amount reported_emergency_loss_by_count; - -/** - * Expected balance in the escrow account. - */ -static struct TALER_Amount total_escrow_balance; - -/** - * Active risk exposure. - */ -static struct TALER_Amount total_risk; - -/** - * Actualized risk (= loss) from recoups. - */ -static struct TALER_Amount total_recoup_loss; - -/** - * Recoups we made on denominations that were not revoked (!?). - */ -static struct TALER_Amount total_irregular_recoups; - -/** - * Total deposit fees earned. - */ -static struct TALER_Amount total_deposit_fee_income; - -/** - * Total melt fees earned. - */ -static struct TALER_Amount total_melt_fee_income; - -/** - * Total refund fees earned. - */ -static struct TALER_Amount total_refund_fee_income; - -/** - * Array of reports about coin operations with bad signatures. - */ -static json_t *report_bad_sig_losses; - -/** - * Total amount lost by operations for which signatures were invalid. - */ -static struct TALER_Amount total_bad_sig_loss; - -/** - * Array of refresh transactions where the /refresh/reveal has not yet - * happened (and may of course never happen). - */ -static json_t *report_refreshs_hanging; - -/** - * Total amount lost by operations for which signatures were invalid. - */ -static struct TALER_Amount total_refresh_hanging; - - -/* ***************************** Report logic **************************** */ - -/** - * Called in case we detect an emergency situation where the exchange - * is paying out a larger amount on a denomination than we issued in - * that denomination. This means that the exchange's private keys - * might have gotten compromised, and that we need to trigger an - * emergency request to all wallets to deposit pending coins for the - * denomination (and as an exchange suffer a huge financial loss). - * - * @param issue denomination key where the loss was detected - * @param risk maximum risk that might have just become real (coins created by this @a issue) - * @param loss actual losses already (actualized before denomination was revoked) - */ -static void -report_emergency_by_amount (const struct TALER_DenominationKeyValidityPS *issue, - const struct TALER_Amount *risk, - const struct TALER_Amount *loss) -{ - report (report_emergencies, - json_pack ("{s:o, s:o, s:o, s:o, s:o, s:o}", - "denompub_hash", - GNUNET_JSON_from_data_auto (&issue->denom_hash), - "denom_risk", - TALER_JSON_from_amount (risk), - "denom_loss", - TALER_JSON_from_amount (loss), - "start", - json_from_time_abs_nbo (issue->start), - "deposit_end", - json_from_time_abs_nbo (issue->expire_deposit), - "value", - TALER_JSON_from_amount_nbo (&issue->value))); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&reported_emergency_risk_by_amount, - &reported_emergency_risk_by_amount, - risk)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&reported_emergency_loss, - &reported_emergency_loss, - loss)); -} - - -/** - * Called in case we detect an emergency situation where the exchange - * is paying out a larger NUMBER of coins of a denomination than we - * issued in that denomination. This means that the exchange's - * private keys might have gotten compromised, and that we need to - * trigger an emergency request to all wallets to deposit pending - * coins for the denomination (and as an exchange suffer a huge - * financial loss). - * - * @param issue denomination key where the loss was detected - * @param num_issued number of coins that were issued - * @param num_known number of coins that have been deposited - * @param risk amount that is at risk - */ -static void -report_emergency_by_count (const struct TALER_DenominationKeyValidityPS *issue, - uint64_t num_issued, - uint64_t num_known, - const struct TALER_Amount *risk) -{ - struct TALER_Amount denom_value; - - report (report_emergencies_by_count, - json_pack ("{s:o, s:I, s:I, s:o, s:o, s:o, s:o}", - "denompub_hash", - GNUNET_JSON_from_data_auto (&issue->denom_hash), - "num_issued", - (json_int_t) num_issued, - "num_known", - (json_int_t) num_known, - "denom_risk", - TALER_JSON_from_amount (risk), - "start", - json_from_time_abs_nbo (issue->start), - "deposit_end", - json_from_time_abs_nbo (issue->expire_deposit), - "value", - TALER_JSON_from_amount_nbo (&issue->value))); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&reported_emergency_risk_by_count, - &reported_emergency_risk_by_count, - risk)); - TALER_amount_ntoh (&denom_value, - &issue->value); - for (uint64_t i = num_issued; i auditor */ - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - exchange, - auditor)); - } - else - { - /* auditor < exchange */ - profitable = -profitable; - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - auditor, - exchange)); - } - report (report_amount_arithmetic_inconsistencies, - json_pack ("{s:s, s:I, s:o, s:o, s:I}", - "operation", operation, - "rowid", (json_int_t) rowid, - "exchange", TALER_JSON_from_amount (exchange), - "auditor", TALER_JSON_from_amount (auditor), - "profitable", (json_int_t) profitable)); - if (0 != profitable) - { - target = (1 == profitable) - ? &total_arithmetic_delta_plus - : &total_arithmetic_delta_minus; - GNUNET_break (GNUNET_OK == - TALER_amount_add (target, - target, - &delta)); - } -} - - -/** - * Report a (serious) inconsistency in the exchange's database. - * - * @param table affected table - * @param rowid affected row, UINT64_MAX if row is missing - * @param diagnostic message explaining the problem - */ -static void -report_row_inconsistency (const char *table, - uint64_t rowid, - const char *diagnostic) -{ - report (report_row_inconsistencies, - json_pack ("{s:s, s:I, s:s}", - "table", table, - "row", (json_int_t) rowid, - "diagnostic", diagnostic)); -} - - -/* ************************* Analyze coins ******************** */ -/* This logic checks that the exchange did the right thing for each - coin, checking deposits, refunds, refresh* and known_coins - tables */ - - -/** - * Summary data we keep per denomination. - */ -struct DenominationSummary -{ - /** - * Total value of outstanding (not deposited) coins issued with this - * denomination key. - */ - struct TALER_Amount denom_balance; - - /** - * Total losses made (once coins deposited exceed - * coins withdrawn and thus the @e denom_balance is - * effectively negative). - */ - struct TALER_Amount denom_loss; - - /** - * Total value of coins issued with this denomination key. - */ - struct TALER_Amount denom_risk; - - /** - * Total value of coins subjected to recoup with this denomination key. - */ - struct TALER_Amount denom_recoup; - - /** - * How many coins (not their amount!) of this denomination - * did the exchange issue overall? - */ - uint64_t num_issued; - - /** - * Denomination key information for this denomination. - */ - const struct TALER_DenominationKeyValidityPS *issue; - - /** - * #GNUNET_YES if this record already existed in the DB. - * Used to decide between insert/update in - * #sync_denomination(). - */ - int in_db; - - /** - * Should we report an emergency for this denomination? - */ - int report_emergency; - - /** - * #GNUNET_YES if this denomination was revoked. - */ - int was_revoked; -}; - - -/** - * Closure for callbacks during #analyze_coins(). - */ -struct CoinContext -{ - - /** - * Map for tracking information about denominations. - */ - struct GNUNET_CONTAINER_MultiHashMap *denom_summaries; - - /** - * Current write/replace offset in the circular @e summaries buffer. - */ - unsigned int summaries_off; - - /** - * Transaction status code. - */ - enum GNUNET_DB_QueryStatus qs; - -}; - - -/** - * Initialize information about denomination from the database. - * - * @param denom_hash hash of the public key of the denomination - * @param[out] ds summary to initialize - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -init_denomination (const struct GNUNET_HashCode *denom_hash, - struct DenominationSummary *ds) -{ - enum GNUNET_DB_QueryStatus qs; - struct TALER_MasterSignatureP msig; - uint64_t rowid; - - qs = adb->get_denomination_balance (adb->cls, - asession, - denom_hash, - &ds->denom_balance, - &ds->denom_loss, - &ds->denom_risk, - &ds->denom_recoup, - &ds->num_issued); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) - { - ds->in_db = GNUNET_YES; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting balance for denomination `%s' is %s\n", - GNUNET_h2s (denom_hash), - TALER_amount2s (&ds->denom_balance)); - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - } - qs = edb->get_denomination_revocation (edb->cls, - esession, - denom_hash, - &msig, - &rowid); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 < qs) - { - /* check revocation signature */ - struct TALER_MasterDenominationKeyRevocationPS rm; - - rm.purpose.purpose = htonl ( - TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED); - rm.purpose.size = htonl (sizeof (rm)); - rm.h_denom_pub = *denom_hash; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify ( - TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED, - &rm.purpose, - &msig.eddsa_signature, - &master_pub.eddsa_pub)) - { - report_row_inconsistency ("denomination revocation table", - rowid, - "revocation signature invalid"); - } - else - { - ds->was_revoked = GNUNET_YES; - } - } - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &ds->denom_balance)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &ds->denom_loss)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &ds->denom_risk)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &ds->denom_recoup)); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting balance for denomination `%s' is %s\n", - GNUNET_h2s (denom_hash), - TALER_amount2s (&ds->denom_balance)); - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; -} - - -/** - * Obtain the denomination summary for the given @a dh - * - * @param cc our execution context - * @param issue denomination key information for @a dh - * @param dh the denomination hash to use for the lookup - * @return NULL on error - */ -static struct DenominationSummary * -get_denomination_summary (struct CoinContext *cc, - const struct TALER_DenominationKeyValidityPS *issue, - const struct GNUNET_HashCode *dh) -{ - struct DenominationSummary *ds; - - ds = GNUNET_CONTAINER_multihashmap_get (cc->denom_summaries, - dh); - if (NULL != ds) - return ds; - ds = GNUNET_new (struct DenominationSummary); - ds->issue = issue; - if (0 > (cc->qs = init_denomination (dh, - ds))) - { - GNUNET_break (0); - GNUNET_free (ds); - return NULL; - } - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (cc->denom_summaries, - dh, - ds, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - return ds; -} - - -/** - * Write information about the current knowledge about a denomination key - * back to the database and update our global reporting data about the - * denomination. Also remove and free the memory of @a value. - * - * @param cls the `struct CoinContext` - * @param denom_hash the hash of the denomination key - * @param value a `struct DenominationSummary` - * @return #GNUNET_OK (continue to iterate) - */ -static int -sync_denomination (void *cls, - const struct GNUNET_HashCode *denom_hash, - void *value) -{ - struct CoinContext *cc = cls; - struct DenominationSummary *ds = value; - const struct TALER_DenominationKeyValidityPS *issue = ds->issue; - struct GNUNET_TIME_Absolute now; - struct GNUNET_TIME_Absolute expire_deposit; - struct GNUNET_TIME_Absolute expire_deposit_grace; - enum GNUNET_DB_QueryStatus qs; - - now = GNUNET_TIME_absolute_get (); - expire_deposit = GNUNET_TIME_absolute_ntoh (issue->expire_deposit); - /* add day grace period to deal with clocks not being perfectly synchronized */ - expire_deposit_grace = GNUNET_TIME_absolute_add (expire_deposit, - DEPOSIT_GRACE_PERIOD); - if (now.abs_value_us > expire_deposit_grace.abs_value_us) - { - /* Denominationkey has expired, book remaining balance of - outstanding coins as revenue; and reduce cc->risk exposure. */ - if (ds->in_db) - qs = adb->del_denomination_balance (adb->cls, - asession, - denom_hash); - else - qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - if ( (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) && - ( (0 != ds->denom_risk.value) || - (0 != ds->denom_risk.fraction) ) ) - { - /* The denomination expired and carried a balance; we can now - book the remaining balance as profit, and reduce our risk - exposure by the accumulated risk of the denomination. */ - if (GNUNET_SYSERR == - TALER_amount_subtract (&total_risk, - &total_risk, - &ds->denom_risk)) - { - /* Holy smokes, our risk assessment was inconsistent! - This is really, really bad. */ - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - } - } - if ( (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) && - ( (0 != ds->denom_balance.value) || - (0 != ds->denom_balance.fraction) ) ) - { - /* book denom_balance coin expiration profits! */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Denomination `%s' expired, booking %s in expiration profits\n", - GNUNET_h2s (denom_hash), - TALER_amount2s (&ds->denom_balance)); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != - (qs = adb->insert_historic_denom_revenue (adb->cls, - asession, - &master_pub, - denom_hash, - expire_deposit, - &ds->denom_balance, - &ds->denom_recoup))) - { - /* Failed to store profits? Bad database */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - } - } - } - else - { - long long cnt; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Final balance for denomination `%s' is %s (%llu)\n", - GNUNET_h2s (denom_hash), - TALER_amount2s (&ds->denom_balance), - (unsigned long long) ds->num_issued); - cnt = edb->count_known_coins (edb->cls, - esession, - denom_hash); - if (0 > cnt) - { - /* Failed to obtain count? Bad database */ - qs = (enum GNUNET_DB_QueryStatus) cnt; - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - } - else - { - if (ds->num_issued < (uint64_t) cnt) - { - report_emergency_by_count (issue, - ds->num_issued, - cnt, - &ds->denom_risk); - } - if (GNUNET_YES == ds->report_emergency) - { - report_emergency_by_amount (issue, - &ds->denom_risk, - &ds->denom_loss); - - } - if (ds->in_db) - qs = adb->update_denomination_balance (adb->cls, - asession, - denom_hash, - &ds->denom_balance, - &ds->denom_loss, - &ds->denom_risk, - &ds->denom_recoup, - ds->num_issued); - else - qs = adb->insert_denomination_balance (adb->cls, - asession, - denom_hash, - &ds->denom_balance, - &ds->denom_loss, - &ds->denom_risk, - &ds->denom_recoup, - ds->num_issued); - } - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - } - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (cc->denom_summaries, - denom_hash, - ds)); - GNUNET_free (ds); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != cc->qs) - return GNUNET_SYSERR; - return GNUNET_OK; -} - - -/** - * Function called with details about all withdraw operations. - * Updates the denomination balance and the overall balance as - * we now have additional coins that have been issued. - * - * Note that the signature was already checked in - * #handle_reserve_out(), so we do not check it again here. - * - * @param cls our `struct CoinContext` - * @param rowid unique serial ID for the refresh session in our DB - * @param h_blind_ev blinded hash of the coin's public key - * @param denom_pub public denomination key of the deposited coin - * @param reserve_pub public key of the reserve - * @param reserve_sig signature over the withdraw operation (verified elsewhere) - * @param execution_date when did the wallet withdraw the coin - * @param amount_with_fee amount that was withdrawn - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -withdraw_cb (void *cls, - uint64_t rowid, - const struct GNUNET_HashCode *h_blind_ev, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_ReservePublicKeyP *reserve_pub, - const struct TALER_ReserveSignatureP *reserve_sig, - struct GNUNET_TIME_Absolute execution_date, - const struct TALER_Amount *amount_with_fee) -{ - struct CoinContext *cc = cls; - struct DenominationSummary *ds; - struct GNUNET_HashCode dh; - const struct TALER_DenominationKeyValidityPS *issue; - struct TALER_Amount value; - enum GNUNET_DB_QueryStatus qs; - - (void) h_blind_ev; - (void) reserve_pub; - (void) reserve_sig; - (void) execution_date; - (void) amount_with_fee; - GNUNET_assert (rowid >= ppc.last_withdraw_serial_id); /* should be monotonically increasing */ - ppc.last_withdraw_serial_id = rowid + 1; - - qs = get_denomination_info (denom_pub, - &issue, - &dh); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("withdraw", - rowid, - "denomination key not found"); - return GNUNET_OK; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - /* This really ought to be a transient DB error. */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - return GNUNET_SYSERR; - } - ds = get_denomination_summary (cc, - issue, - &dh); - if (NULL == ds) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - TALER_amount_ntoh (&value, - &issue->value); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Issued coin in denomination `%s' of total value %s\n", - GNUNET_h2s (&dh), - TALER_amount2s (&value)); - ds->num_issued++; - if (GNUNET_OK != - TALER_amount_add (&ds->denom_balance, - &ds->denom_balance, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "New balance of denomination `%s' is %s\n", - GNUNET_h2s (&dh), - TALER_amount2s (&ds->denom_balance)); - if (GNUNET_OK != - TALER_amount_add (&total_escrow_balance, - &total_escrow_balance, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&total_risk, - &total_risk, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&ds->denom_risk, - &ds->denom_risk, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - return GNUNET_OK; -} - - -/** - * Closure for #reveal_data_cb(). - */ -struct RevealContext -{ - - /** - * Denomination public keys of the new coins. - */ - struct TALER_DenominationPublicKey *new_dps; - - /** - * Size of the @a new_dp and @a new_dps arrays. - */ - unsigned int num_freshcoins; -}; - - -/** - * Function called with information about a refresh order. - * - * @param cls closure - * @param num_freshcoins size of the @a rrcs array - * @param rrcs array of @a num_freshcoins information about coins to be created - * @param num_tprivs number of entries in @a tprivs, should be #TALER_CNC_KAPPA - 1 - * @param tprivs array of @e num_tprivs transfer private keys - * @param tp transfer public key information - */ -static void -reveal_data_cb (void *cls, - uint32_t num_freshcoins, - const struct TALER_EXCHANGEDB_RefreshRevealedCoin *rrcs, - unsigned int num_tprivs, - const struct TALER_TransferPrivateKeyP *tprivs, - const struct TALER_TransferPublicKeyP *tp) -{ - struct RevealContext *rctx = cls; - - (void) num_tprivs; - (void) tprivs; - (void) tp; - rctx->num_freshcoins = num_freshcoins; - rctx->new_dps = GNUNET_new_array (num_freshcoins, - struct TALER_DenominationPublicKey); - for (unsigned int i = 0; inew_dps[i].rsa_public_key - = GNUNET_CRYPTO_rsa_public_key_dup (rrcs[i].denom_pub.rsa_public_key); -} - - -/** - * Check that the @a coin_pub is a known coin with a proper - * signature for denominatinon @a denom_pub. If not, report - * a loss of @a loss_potential. - * - * @param coin_pub public key of a coin - * @param denom_pub expected denomination of the coin - * @param loss_potential how big could the loss be if the coin is - * not properly signed - * @return database transaction status, on success - * #GNUNET_DB_STATUS_SUCCESS_ONE_RESULT - */ -static enum GNUNET_DB_QueryStatus -check_known_coin (const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_Amount *loss_potential) -{ - struct TALER_CoinPublicInfo ci; - enum GNUNET_DB_QueryStatus qs; - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Checking denomination signature on %s\n", - TALER_B2S (coin_pub)); - qs = edb->get_known_coin (edb->cls, - esession, - coin_pub, - &ci); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (GNUNET_YES != - TALER_test_coin_valid (&ci, - denom_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "known-coin", - "row", (json_int_t) -1, - "loss", TALER_JSON_from_amount (loss_potential), - "key_pub", GNUNET_JSON_from_data_auto (coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - loss_potential)); - - } - GNUNET_CRYPTO_rsa_signature_free (ci.denom_sig.rsa_signature); - return qs; -} - - -/** - * Function called with details about coins that were melted, with the - * goal of auditing the refresh's execution. Verifies the signature - * and updates our information about coins outstanding (the old coin's - * denomination has less, the fresh coins increased outstanding - * balances). - * - * @param cls closure - * @param rowid unique serial ID for the refresh session in our DB - * @param denom_pub denomination public key of @a coin_pub - * @param coin_pub public key of the coin - * @param coin_sig signature from the coin - * @param amount_with_fee amount that was deposited including fee - * @param noreveal_index which index was picked by the exchange in cut-and-choose - * @param rc what is the refresh commitment - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -refresh_session_cb (void *cls, - uint64_t rowid, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_CoinSpendSignatureP *coin_sig, - const struct TALER_Amount *amount_with_fee, - uint32_t noreveal_index, - const struct TALER_RefreshCommitmentP *rc) -{ - struct CoinContext *cc = cls; - struct TALER_RefreshMeltCoinAffirmationPS rmc; - const struct TALER_DenominationKeyValidityPS *issue; - struct DenominationSummary *dso; - struct TALER_Amount amount_without_fee; - struct TALER_Amount tmp; - enum GNUNET_DB_QueryStatus qs; - - (void) noreveal_index; - GNUNET_assert (rowid >= ppc.last_melt_serial_id); /* should be monotonically increasing */ - ppc.last_melt_serial_id = rowid + 1; - - qs = get_denomination_info (denom_pub, - &issue, - NULL); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("melt", - rowid, - "denomination key not found"); - return GNUNET_OK; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - return GNUNET_SYSERR; - } - qs = check_known_coin (coin_pub, - denom_pub, - amount_with_fee); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - return GNUNET_SYSERR; - } - - /* verify melt signature */ - rmc.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_MELT); - rmc.purpose.size = htonl (sizeof (rmc)); - rmc.rc = *rc; - TALER_amount_hton (&rmc.amount_with_fee, - amount_with_fee); - rmc.melt_fee = issue->fee_refresh; - rmc.coin_pub = *coin_pub; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_MELT, - &rmc.purpose, - &coin_sig->eddsa_signature, - &coin_pub->eddsa_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "melt", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount_with_fee), - "key_pub", GNUNET_JSON_from_data_auto (coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount_with_fee)); - return GNUNET_OK; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Melting coin %s in denomination `%s' of value %s\n", - TALER_B2S (coin_pub), - GNUNET_h2s (&issue->denom_hash), - TALER_amount2s (amount_with_fee)); - - { - struct RevealContext reveal_ctx; - struct TALER_Amount refresh_cost; - int err; - - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount_with_fee->currency, - &refresh_cost)); - memset (&reveal_ctx, - 0, - sizeof (reveal_ctx)); - qs = edb->get_refresh_reveal (edb->cls, - esession, - rc, - &reveal_data_cb, - &reveal_ctx); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if ( (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) || - (0 == reveal_ctx.num_freshcoins) ) - { - /* This can happen if /refresh/reveal was not yet called or only - with invalid data, even if the exchange is correctly - operating. We still report it. */ - report (report_refreshs_hanging, - json_pack ("{s:I, s:o, s:o}", - "row", (json_int_t) rowid, - "amount", TALER_JSON_from_amount (amount_with_fee), - "coin_pub", GNUNET_JSON_from_data_auto (coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_refresh_hanging, - &total_refresh_hanging, - amount_with_fee)); - return GNUNET_OK; - } - - { - const struct TALER_DenominationKeyValidityPS *new_issues[reveal_ctx. - num_freshcoins]; - - /* Update outstanding amounts for all new coin's denominations, and check - that the resulting amounts are consistent with the value being refreshed. */ - err = GNUNET_OK; - for (unsigned int i = 0; iqs = qs; - err = GNUNET_SYSERR; /* terminate, return GNUNET_SYSERR */ - } - GNUNET_CRYPTO_rsa_public_key_free ( - reveal_ctx.new_dps[i].rsa_public_key); - reveal_ctx.new_dps[i].rsa_public_key = NULL; - } - GNUNET_free (reveal_ctx.new_dps); - reveal_ctx.new_dps = NULL; - - if (GNUNET_OK != err) - return (GNUNET_SYSERR == err) ? GNUNET_SYSERR : GNUNET_OK; - - /* calculate total refresh cost */ - for (unsigned int i = 0; ifee_withdraw); - TALER_amount_ntoh (&value, - &new_issues[i]->value); - if ( (GNUNET_OK != - TALER_amount_add (&refresh_cost, - &refresh_cost, - &fee)) || - (GNUNET_OK != - TALER_amount_add (&refresh_cost, - &refresh_cost, - &value)) ) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - } - - /* compute contribution of old coin */ - { - struct TALER_Amount melt_fee; - - TALER_amount_ntoh (&melt_fee, - &issue->fee_refresh); - if (GNUNET_OK != - TALER_amount_subtract (&amount_without_fee, - amount_with_fee, - &melt_fee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - } - - /* check old coin covers complete expenses */ - if (1 == TALER_amount_cmp (&refresh_cost, - &amount_without_fee)) - { - /* refresh_cost > amount_without_fee */ - report_amount_arithmetic_inconsistency ("melt (fee)", - rowid, - &amount_without_fee, - &refresh_cost, - -1); - return GNUNET_OK; - } - - /* update outstanding denomination amounts */ - for (unsigned int i = 0; idenom_hash); - if (NULL == dsi) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - TALER_amount_ntoh (&value, - &new_issues[i]->value); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Created fresh coin in denomination `%s' of value %s\n", - GNUNET_h2s (&new_issues[i]->denom_hash), - TALER_amount2s (&value)); - dsi->num_issued++; - if (GNUNET_OK != - TALER_amount_add (&dsi->denom_balance, - &dsi->denom_balance, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&dsi->denom_risk, - &dsi->denom_risk, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "New balance of denomination `%s' is %s\n", - GNUNET_h2s (&new_issues[i]->denom_hash), - TALER_amount2s (&dsi->denom_balance)); - if (GNUNET_OK != - TALER_amount_add (&total_escrow_balance, - &total_escrow_balance, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&total_risk, - &total_risk, - &value)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - } - } - } - - /* update old coin's denomination balance */ - dso = get_denomination_summary (cc, - issue, - &issue->denom_hash); - if (NULL == dso) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - if (GNUNET_SYSERR == - TALER_amount_subtract (&tmp, - &dso->denom_balance, - amount_with_fee)) - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&dso->denom_loss, - &dso->denom_loss, - amount_with_fee)); - dso->report_emergency = GNUNET_YES; - } - else - { - dso->denom_balance = tmp; - } - if (-1 == TALER_amount_cmp (&total_escrow_balance, - amount_with_fee)) - { - /* This can theoretically happen if for example the exchange - never issued any coins (i.e. escrow balance is zero), but - accepted a forged coin (i.e. emergency situation after - private key compromise). In that case, we cannot even - subtract the profit we make from the fee from the escrow - balance. Tested as part of test-auditor.sh, case #18 */report_amount_arithmetic_inconsistency ( - "subtracting refresh fee from escrow balance", - rowid, - &total_escrow_balance, - amount_with_fee, - 0); - } - else - { - GNUNET_assert (GNUNET_SYSERR != - TALER_amount_subtract (&total_escrow_balance, - &total_escrow_balance, - amount_with_fee)); - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "New balance of denomination `%s' after melt is %s\n", - GNUNET_h2s (&issue->denom_hash), - TALER_amount2s (&dso->denom_balance)); - - /* update global melt fees */ - { - struct TALER_Amount rfee; - - TALER_amount_ntoh (&rfee, - &issue->fee_refresh); - if (GNUNET_OK != - TALER_amount_add (&total_melt_fee_income, - &total_melt_fee_income, - &rfee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - } - - /* We're good! */ - return GNUNET_OK; -} - - -/** - * Function called with details about deposits that have been made, - * with the goal of auditing the deposit's execution. - * - * @param cls closure - * @param rowid unique serial ID for the deposit in our DB - * @param timestamp when did the deposit happen - * @param merchant_pub public key of the merchant - * @param denom_pub denomination public key of @a coin_pub - * @param coin_pub public key of the coin - * @param coin_sig signature from the coin - * @param amount_with_fee amount that was deposited including fee - * @param h_contract_terms hash of the proposal data known to merchant and customer - * @param refund_deadline by which the merchant adviced that he might want - * to get a refund - * @param wire_deadline by which the merchant adviced that he would like the - * wire transfer to be executed - * @param receiver_wire_account wire details for the merchant, NULL from iterate_matching_deposits() - * @param done flag set if the deposit was already executed (or not) - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -deposit_cb (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute timestamp, - const struct TALER_MerchantPublicKeyP *merchant_pub, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_CoinSpendSignatureP *coin_sig, - const struct TALER_Amount *amount_with_fee, - const struct GNUNET_HashCode *h_contract_terms, - struct GNUNET_TIME_Absolute refund_deadline, - struct GNUNET_TIME_Absolute wire_deadline, - const json_t *receiver_wire_account, - int done) -{ - struct CoinContext *cc = cls; - const struct TALER_DenominationKeyValidityPS *issue; - struct DenominationSummary *ds; - struct TALER_DepositRequestPS dr; - struct TALER_Amount tmp; - enum GNUNET_DB_QueryStatus qs; - - (void) wire_deadline; - (void) done; - GNUNET_assert (rowid >= ppc.last_deposit_serial_id); /* should be monotonically increasing */ - ppc.last_deposit_serial_id = rowid + 1; - - qs = get_denomination_info (denom_pub, - &issue, - NULL); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("deposits", - rowid, - "denomination key not found"); - return GNUNET_OK; - } - - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - return GNUNET_SYSERR; - } - qs = check_known_coin (coin_pub, - denom_pub, - amount_with_fee); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - return GNUNET_SYSERR; - } - - /* Verify deposit signature */ - dr.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_DEPOSIT); - dr.purpose.size = htonl (sizeof (dr)); - dr.h_contract_terms = *h_contract_terms; - if (GNUNET_OK != - TALER_JSON_merchant_wire_signature_hash (receiver_wire_account, - &dr.h_wire)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "deposit", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount_with_fee), - "key_pub", GNUNET_JSON_from_data_auto (coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount_with_fee)); - return GNUNET_OK; - } - dr.timestamp = GNUNET_TIME_absolute_hton (timestamp); - dr.refund_deadline = GNUNET_TIME_absolute_hton (refund_deadline); - TALER_amount_hton (&dr.amount_with_fee, - amount_with_fee); - dr.deposit_fee = issue->fee_deposit; - dr.merchant = *merchant_pub; - dr.coin_pub = *coin_pub; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_DEPOSIT, - &dr.purpose, - &coin_sig->eddsa_signature, - &coin_pub->eddsa_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "deposit", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount_with_fee), - "key_pub", GNUNET_JSON_from_data_auto (coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount_with_fee)); - return GNUNET_OK; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Deposited coin %s in denomination `%s' of value %s\n", - TALER_B2S (coin_pub), - GNUNET_h2s (&issue->denom_hash), - TALER_amount2s (amount_with_fee)); - - /* update old coin's denomination balance */ - ds = get_denomination_summary (cc, - issue, - &issue->denom_hash); - if (NULL == ds) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - if (GNUNET_SYSERR == - TALER_amount_subtract (&tmp, - &ds->denom_balance, - amount_with_fee)) - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&ds->denom_loss, - &ds->denom_loss, - amount_with_fee)); - ds->report_emergency = GNUNET_YES; - } - else - { - ds->denom_balance = tmp; - } - - if (-1 == TALER_amount_cmp (&total_escrow_balance, - amount_with_fee)) - { - /* This can theoretically happen if for example the exchange - never issued any coins (i.e. escrow balance is zero), but - accepted a forged coin (i.e. emergency situation after - private key compromise). In that case, we cannot even - subtract the profit we make from the fee from the escrow - balance. Tested as part of test-auditor.sh, case #18 */report_amount_arithmetic_inconsistency ( - "subtracting deposit fee from escrow balance", - rowid, - &total_escrow_balance, - amount_with_fee, - 0); - } - else - { - GNUNET_assert (GNUNET_SYSERR != - TALER_amount_subtract (&total_escrow_balance, - &total_escrow_balance, - amount_with_fee)); - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "New balance of denomination `%s' after deposit is %s\n", - GNUNET_h2s (&issue->denom_hash), - TALER_amount2s (&ds->denom_balance)); - - /* update global up melt fees */ - { - struct TALER_Amount dfee; - - TALER_amount_ntoh (&dfee, - &issue->fee_deposit); - if (GNUNET_OK != - TALER_amount_add (&total_deposit_fee_income, - &total_deposit_fee_income, - &dfee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - } - - return GNUNET_OK; -} - - -/** - * Function called with details about coins that were refunding, - * with the goal of auditing the refund's execution. Adds the - * refunded amount back to the outstanding balance of the respective - * denomination. - * - * @param cls closure - * @param rowid unique serial ID for the refund in our DB - * @param denom_pub denomination public key of @a coin_pub - * @param coin_pub public key of the coin - * @param merchant_pub public key of the merchant - * @param merchant_sig signature of the merchant - * @param h_contract_terms hash of the proposal data known to merchant and customer - * @param rtransaction_id refund transaction ID chosen by the merchant - * @param amount_with_fee amount that was deposited including fee - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -refund_cb (void *cls, - uint64_t rowid, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_MerchantPublicKeyP *merchant_pub, - const struct TALER_MerchantSignatureP *merchant_sig, - const struct GNUNET_HashCode *h_contract_terms, - uint64_t rtransaction_id, - const struct TALER_Amount *amount_with_fee) -{ - struct CoinContext *cc = cls; - const struct TALER_DenominationKeyValidityPS *issue; - struct DenominationSummary *ds; - struct TALER_RefundRequestPS rr; - struct TALER_Amount amount_without_fee; - struct TALER_Amount refund_fee; - enum GNUNET_DB_QueryStatus qs; - - GNUNET_assert (rowid >= ppc.last_refund_serial_id); /* should be monotonically increasing */ - ppc.last_refund_serial_id = rowid + 1; - - qs = get_denomination_info (denom_pub, - &issue, - NULL); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("refunds", - rowid, - "denomination key not found"); - return GNUNET_SYSERR; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return GNUNET_SYSERR; - } - - /* verify refund signature */ - rr.purpose.purpose = htonl (TALER_SIGNATURE_MERCHANT_REFUND); - rr.purpose.size = htonl (sizeof (rr)); - rr.h_contract_terms = *h_contract_terms; - rr.coin_pub = *coin_pub; - rr.merchant = *merchant_pub; - rr.rtransaction_id = GNUNET_htonll (rtransaction_id); - TALER_amount_hton (&rr.refund_amount, - amount_with_fee); - rr.refund_fee = issue->fee_refund; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_MERCHANT_REFUND, - &rr.purpose, - &merchant_sig->eddsa_sig, - &merchant_pub->eddsa_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "refund", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount_with_fee), - "key_pub", GNUNET_JSON_from_data_auto (merchant_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount_with_fee)); - return GNUNET_OK; - } - - TALER_amount_ntoh (&refund_fee, - &issue->fee_refund); - if (GNUNET_OK != - TALER_amount_subtract (&amount_without_fee, - amount_with_fee, - &refund_fee)) - { - report_amount_arithmetic_inconsistency ("refund (fee)", - rowid, - &amount_without_fee, - &refund_fee, - -1); - return GNUNET_OK; - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Refunding coin %s in denomination `%s' value %s\n", - TALER_B2S (coin_pub), - GNUNET_h2s (&issue->denom_hash), - TALER_amount2s (amount_with_fee)); - - /* update coin's denomination balance */ - ds = get_denomination_summary (cc, - issue, - &issue->denom_hash); - if (NULL == ds) - { - GNUNET_break (0); - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&ds->denom_balance, - &ds->denom_balance, - &amount_without_fee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&ds->denom_risk, - &ds->denom_risk, - &amount_without_fee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&total_escrow_balance, - &total_escrow_balance, - &amount_without_fee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - if (GNUNET_OK != - TALER_amount_add (&total_risk, - &total_risk, - &amount_without_fee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "New balance of denomination `%s' after refund is %s\n", - GNUNET_h2s (&issue->denom_hash), - TALER_amount2s (&ds->denom_balance)); - - /* update total refund fee balance */ - if (GNUNET_OK != - TALER_amount_add (&total_refund_fee_income, - &total_refund_fee_income, - &refund_fee)) - { - GNUNET_break (0); - cc->qs = GNUNET_DB_STATUS_HARD_ERROR; - return GNUNET_SYSERR; - } - - return GNUNET_OK; -} - - -/** - * Check that the recoup operation was properly initiated by a coin - * and update the denomination's losses accordingly. - * - * @param cc the context with details about the coin - * @param rowid row identifier used to uniquely identify the recoup operation - * @param amount how much should be added back to the reserve - * @param coin public information about the coin - * @param denom_pub public key of the denomionation of @a coin - * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP - * @param coin_blind blinding factor used to blind the coin - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -check_recoup (struct CoinContext *cc, - uint64_t rowid, - const struct TALER_Amount *amount, - const struct TALER_CoinPublicInfo *coin, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendSignatureP *coin_sig, - const struct TALER_DenominationBlindingKeyP *coin_blind) -{ - struct TALER_RecoupRequestPS pr; - struct DenominationSummary *ds; - enum GNUNET_DB_QueryStatus qs; - const struct TALER_DenominationKeyValidityPS *issue; - - if (GNUNET_OK != - TALER_test_coin_valid (coin, - denom_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "recoup", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount), - "key_pub", GNUNET_JSON_from_data_auto ( - &pr.h_denom_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount)); - } - qs = get_denomination_info (denom_pub, - &issue, - &pr.h_denom_pub); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("recoup", - rowid, - "denomination key not found"); - return GNUNET_OK; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - /* The key not existing should be prevented by foreign key constraints, - so must be a transient DB error. */ - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - cc->qs = qs; - return GNUNET_SYSERR; - } - pr.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_RECOUP); - pr.purpose.size = htonl (sizeof (pr)); - pr.coin_pub = coin->coin_pub; - pr.coin_blind = *coin_blind; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_RECOUP, - &pr.purpose, - &coin_sig->eddsa_signature, - &coin->coin_pub.eddsa_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "recoup", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount), - "coin_pub", GNUNET_JSON_from_data_auto ( - &coin->coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount)); - return GNUNET_OK; - } - ds = get_denomination_summary (cc, - issue, - &issue->denom_hash); - if (GNUNET_NO == ds->was_revoked) - { - /* Woopsie, we allowed recoup on non-revoked denomination!? */ - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "recoup (denomination not revoked)", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount), - "coin_pub", GNUNET_JSON_from_data_auto ( - &coin->coin_pub))); - } - GNUNET_break (GNUNET_OK == - TALER_amount_add (&ds->denom_recoup, - &ds->denom_recoup, - amount)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_recoup_loss, - &total_recoup_loss, - amount)); - return GNUNET_OK; -} - - -/** - * Function called about recoups the exchange has to perform. - * - * @param cls a `struct CoinContext *` - * @param rowid row identifier used to uniquely identify the recoup operation - * @param timestamp when did we receive the recoup request - * @param amount how much should be added back to the reserve - * @param reserve_pub public key of the reserve - * @param coin public information about the coin - * @param denom_pub denomination public key of @a coin - * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP - * @param coin_blind blinding factor used to blind the coin - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -recoup_cb (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute timestamp, - const struct TALER_Amount *amount, - const struct TALER_ReservePublicKeyP *reserve_pub, - const struct TALER_CoinPublicInfo *coin, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendSignatureP *coin_sig, - const struct TALER_DenominationBlindingKeyP *coin_blind) -{ - struct CoinContext *cc = cls; - - (void) timestamp; - (void) reserve_pub; - return check_recoup (cc, - rowid, - amount, - coin, - denom_pub, - coin_sig, - coin_blind); -} - - -/** - * Function called about recoups on refreshed coins the exchange has to - * perform. - * - * @param cls a `struct CoinContext *` - * @param rowid row identifier used to uniquely identify the recoup operation - * @param timestamp when did we receive the recoup request - * @param amount how much should be added back to the reserve - * @param old_coin_pub original coin that was refreshed to create @a coin - * @param coin public information about the coin - * @param denom_pub denomination public key of @a coin - * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP - * @param coin_blind blinding factor used to blind the coin - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -recoup_refresh_cb (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute timestamp, - const struct TALER_Amount *amount, - const struct TALER_CoinSpendPublicKeyP *old_coin_pub, - const struct TALER_CoinPublicInfo *coin, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendSignatureP *coin_sig, - const struct TALER_DenominationBlindingKeyP *coin_blind) -{ - struct CoinContext *cc = cls; - - (void) timestamp; - (void) old_coin_pub; - return check_recoup (cc, - rowid, - amount, - coin, - denom_pub, - coin_sig, - coin_blind); -} - - -/** - * Analyze the exchange's processing of coins. - * - * @param cls closure - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -analyze_coins (void *cls) -{ - struct CoinContext cc; - enum GNUNET_DB_QueryStatus qs; - enum GNUNET_DB_QueryStatus qsx; - enum GNUNET_DB_QueryStatus qsp; - - (void) cls; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Analyzing coins\n"); - qsp = adb->get_auditor_progress_coin (adb->cls, - asession, - &master_pub, - &ppc); - if (0 > qsp) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); - return qsp; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) - { - GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, - "First analysis using this auditor, starting from scratch\n"); - } - else - { - ppc_start = ppc; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Resuming coin audit at %llu/%llu/%llu/%llu/%llu\n", - (unsigned long long) ppc.last_deposit_serial_id, - (unsigned long long) ppc.last_melt_serial_id, - (unsigned long long) ppc.last_refund_serial_id, - (unsigned long long) ppc.last_withdraw_serial_id, - (unsigned long long) ppc.last_recoup_refresh_serial_id); - } - - /* setup 'cc' */ - cc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - cc.denom_summaries = GNUNET_CONTAINER_multihashmap_create (256, - GNUNET_NO); - qsx = adb->get_balance_summary (adb->cls, - asession, - &master_pub, - &total_escrow_balance, - &total_deposit_fee_income, - &total_melt_fee_income, - &total_refund_fee_income, - &total_risk, - &total_recoup_loss, - &total_irregular_recoups); - if (0 > qsx) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); - return qsx; - } - - /* process withdrawals */ - if (0 > - (qs = edb->select_withdrawals_above_serial_id (edb->cls, - esession, - ppc. - last_withdraw_serial_id, - &withdraw_cb, - &cc)) ) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 > cc.qs) - return cc.qs; - - /* process refunds */ - if (0 > - (qs = edb->select_refunds_above_serial_id (edb->cls, - esession, - ppc.last_refund_serial_id, - &refund_cb, - &cc))) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 > cc.qs) - return cc.qs; - - /* process refreshs */ - if (0 > - (qs = edb->select_refreshes_above_serial_id (edb->cls, - esession, - ppc.last_melt_serial_id, - &refresh_session_cb, - &cc))) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 > cc.qs) - return cc.qs; - - /* process deposits */ - if (0 > - (qs = edb->select_deposits_above_serial_id (edb->cls, - esession, - ppc.last_deposit_serial_id, - &deposit_cb, - &cc))) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 > cc.qs) - return cc.qs; - - /* process recoups */ - if (0 > - (qs = edb->select_recoup_above_serial_id (edb->cls, - esession, - ppc.last_recoup_serial_id, - &recoup_cb, - &cc))) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 > cc.qs) - return cc.qs; - if (0 > - (qs = edb->select_recoup_refresh_above_serial_id (edb->cls, - esession, - ppc. - last_recoup_refresh_serial_id, - &recoup_refresh_cb, - &cc))) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (0 > cc.qs) - return cc.qs; - - /* sync 'cc' back to disk */ - cc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - GNUNET_CONTAINER_multihashmap_iterate (cc.denom_summaries, - &sync_denomination, - &cc); - GNUNET_CONTAINER_multihashmap_destroy (cc.denom_summaries); - if (0 > cc.qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == cc.qs); - return cc.qs; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsx) - qs = adb->update_balance_summary (adb->cls, - asession, - &master_pub, - &total_escrow_balance, - &total_deposit_fee_income, - &total_melt_fee_income, - &total_refund_fee_income, - &total_risk, - &total_recoup_loss, - &total_irregular_recoups); - else - qs = adb->insert_balance_summary (adb->cls, - asession, - &master_pub, - &total_escrow_balance, - &total_deposit_fee_income, - &total_melt_fee_income, - &total_refund_fee_income, - &total_risk, - &total_recoup_loss, - &total_irregular_recoups); - if (0 >= qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) - qs = adb->update_auditor_progress_coin (adb->cls, - asession, - &master_pub, - &ppc); - else - qs = adb->insert_auditor_progress_coin (adb->cls, - asession, - &master_pub, - &ppc); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to update auditor DB, not recording progress\n"); - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Concluded coin audit step at %llu/%llu/%llu/%llu/%llu\n"), - (unsigned long long) ppc.last_deposit_serial_id, - (unsigned long long) ppc.last_melt_serial_id, - (unsigned long long) ppc.last_refund_serial_id, - (unsigned long long) ppc.last_withdraw_serial_id, - (unsigned long long) ppc.last_recoup_refresh_serial_id); - return qs; -} - - -/** - * Main function that will be run. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - json_t *report; - - (void) cls; - (void) args; - (void) cfgfile; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Launching auditor\n"); - if (GNUNET_OK != - setup_globals (c)) - { - global_ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting audit\n"); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &reported_emergency_loss)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &reported_emergency_risk_by_amount)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &reported_emergency_risk_by_count)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &reported_emergency_loss_by_count)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_escrow_balance)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_risk)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_recoup_loss)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_irregular_recoups)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_deposit_fee_income)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_melt_fee_income)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_refund_fee_income)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_arithmetic_delta_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_arithmetic_delta_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_sig_loss)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_refresh_hanging)); - GNUNET_assert (NULL != - (report_emergencies = json_array ())); - GNUNET_assert (NULL != - (report_emergencies_by_count = json_array ())); - GNUNET_assert (NULL != - (report_row_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_amount_arithmetic_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_bad_sig_losses = json_array ())); - GNUNET_assert (NULL != - (report_refreshs_hanging = json_array ())); - if (GNUNET_OK != - setup_sessions_and_run (&analyze_coins, - NULL)) - { - global_ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Audit complete\n"); - report = json_pack ("{s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:I, s:I, s:I, s:I, s:I," - " s:I, s:I, s:I, s:I, s:I," - " s:I, s:I, s:o, s:o, s:o}", - /* Block #1 */ - "total_escrow_balance", - TALER_JSON_from_amount (&total_escrow_balance), - "total_active_risk", - TALER_JSON_from_amount (&total_risk), - "total_deposit_fee_income", - TALER_JSON_from_amount (&total_deposit_fee_income), - "total_melt_fee_income", - TALER_JSON_from_amount (&total_melt_fee_income), - "total_refund_fee_income", - TALER_JSON_from_amount (&total_refund_fee_income), - /* Block #2 */ - /* Tested in test-auditor.sh #18 */ - "emergencies", - report_emergencies, - /* Tested in test-auditor.sh #18 */ - "emergencies_risk_by_amount", - TALER_JSON_from_amount ( - &reported_emergency_risk_by_amount), - /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ - "bad_sig_losses", - report_bad_sig_losses, - /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ - "total_bad_sig_loss", - TALER_JSON_from_amount (&total_bad_sig_loss), - /* Tested in test-auditor.sh #14/#15 */ - "row_inconsistencies", - report_row_inconsistencies, - /* Block #3 */ - "amount_arithmetic_inconsistencies", - report_amount_arithmetic_inconsistencies, - "total_arithmetic_delta_plus", - TALER_JSON_from_amount (&total_arithmetic_delta_plus), - "total_arithmetic_delta_minus", - TALER_JSON_from_amount (&total_arithmetic_delta_minus), - /* Tested in test-auditor.sh #12 */ - "total_refresh_hanging", - TALER_JSON_from_amount (&total_refresh_hanging), - /* Tested in test-auditor.sh #12 */ - "refresh_hanging", - report_refreshs_hanging, - /* Block #4 */ - "total_recoup_loss", - TALER_JSON_from_amount (&total_recoup_loss), - /* Tested in test-auditor.sh #18 */ - "emergencies_by_count", - report_emergencies_by_count, - /* Tested in test-auditor.sh #18 */ - "emergencies_risk_by_count", - TALER_JSON_from_amount ( - &reported_emergency_risk_by_count), - /* Tested in test-auditor.sh #18 */ - "emergencies_loss", - TALER_JSON_from_amount (&reported_emergency_loss), - /* Tested in test-auditor.sh #18 */ - "emergencies_loss_by_count", - TALER_JSON_from_amount ( - &reported_emergency_loss_by_count), - /* Block #5 */ - "start_ppc_withdraw_serial_id", - (json_int_t) ppc_start.last_withdraw_serial_id, - "start_ppc_deposit_serial_id", - (json_int_t) ppc_start.last_deposit_serial_id, - "start_ppc_melt_serial_id", - (json_int_t) ppc_start.last_melt_serial_id, - "start_ppc_refund_serial_id", - (json_int_t) ppc_start.last_refund_serial_id, - "start_ppc_recoup_serial_id", - (json_int_t) ppc_start.last_recoup_serial_id, - /* Block #6 */ - "start_ppc_recoup_refresh_serial_id", - (json_int_t) ppc_start.last_recoup_refresh_serial_id, - "end_ppc_withdraw_serial_id", - (json_int_t) ppc.last_withdraw_serial_id, - "end_ppc_deposit_serial_id", - (json_int_t) ppc.last_deposit_serial_id, - "end_ppc_melt_serial_id", - (json_int_t) ppc.last_melt_serial_id, - "end_ppc_refund_serial_id", - (json_int_t) ppc.last_refund_serial_id, - /* Block #7 */ - "end_ppc_recoup_serial_id", - (json_int_t) ppc.last_recoup_serial_id, - "end_ppc_recoup_refresh_serial_id", - (json_int_t) ppc.last_recoup_refresh_serial_id, - "auditor_start_time", json_string ( - GNUNET_STRINGS_absolute_time_to_string (start_time)), - "auditor_end_time", json_string ( - GNUNET_STRINGS_absolute_time_to_string ( - GNUNET_TIME_absolute_get ())), - "total_irregular_recoups", - TALER_JSON_from_amount (&total_irregular_recoups) - ); - GNUNET_break (NULL != report); - finish_report (report); -} - - -/** - * The main function of the database initialization tool. - * Used to initialize the Taler Exchange's database. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, - char *const *argv) -{ - const struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_base32_auto ('m', - "exchange-key", - "KEY", - "public key of the exchange (Crockford base32 encoded)", - &master_pub), - GNUNET_GETOPT_option_flag ('r', - "restart", - "restart audit from the beginning (required on first run)", - &restart), - GNUNET_GETOPT_option_timetravel ('T', - "timetravel"), - GNUNET_GETOPT_OPTION_END - }; - - /* force linker to link against libtalerutil; if we do - not do this, the linker may "optimize" libtalerutil - away and skip #TALER_OS_init(), which we do need */ - (void) TALER_project_data_default (); - GNUNET_assert (GNUNET_OK == - GNUNET_log_setup ("taler-auditor", - "MESSAGE", - NULL)); - if (GNUNET_OK != - GNUNET_PROGRAM_run (argc, - argv, - "taler-auditor", - "Audit Taler exchange database", - options, - &run, - NULL)) - return 1; - return global_ret; -} - - -/* end of taler-auditor.c */ diff --git a/src/auditor/taler-auditor-deposits.c b/src/auditor/taler-auditor-deposits.c deleted file mode 100644 index ac8a0b624..000000000 --- a/src/auditor/taler-auditor-deposits.c +++ /dev/null @@ -1,360 +0,0 @@ -/* - This file is part of TALER - Copyright (C) 2016-2020 Taler Systems SA - - TALER is free software; you can redistribute it and/or modify it under the - terms of the GNU Affero Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - TALER is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. - - You should have received a copy of the GNU Affero Public License along with - TALER; see the file COPYING. If not, see -*/ -/** - * @file auditor/taler-auditor-deposits.c - * @brief audits an exchange database for deposit confirmation consistency - * @author Christian Grothoff - */ -#include "platform.h" -#include -#include "taler_auditordb_plugin.h" -#include "taler_exchangedb_lib.h" -#include "taler_json_lib.h" -#include "taler_bank_service.h" -#include "taler_signatures.h" -#include "report-lib.h" - - -/** - * Return value from main(). - */ -static int global_ret; - -/** - * Array of reports about missing deposit confirmations. - */ -static json_t *report_deposit_confirmation_inconsistencies; - -/** - * Total number of deposit confirmations that we did not get. - */ -static json_int_t number_missed_deposit_confirmations; - -/** - * Total amount involved in deposit confirmations that we did not get. - */ -static struct TALER_Amount total_missed_deposit_confirmations; - - -/* *************************** Analysis of deposit-confirmations ********** */ - -/** - * Closure for #test_dc. - */ -struct DepositConfirmationContext -{ - - /** - * How many deposit confirmations did we NOT find in the #edb? - */ - unsigned long long missed_count; - - /** - * What is the total amount missing? - */ - struct TALER_Amount missed_amount; - - /** - * Lowest SerialID of the first coin we missed? (This is where we - * should resume next time). - */ - uint64_t first_missed_coin_serial; - - /** - * Lowest SerialID of the first coin we missed? (This is where we - * should resume next time). - */ - uint64_t last_seen_coin_serial; - - /** - * Success or failure of (exchange) database operations within - * #test_dc. - */ - enum GNUNET_DB_QueryStatus qs; - -}; - - -/** - * Given a deposit confirmation from #adb, check that it is also - * in #edb. Update the deposit confirmation context accordingly. - * - * @param cls our `struct DepositConfirmationContext` - * @param serial_id row of the @a dc in the database - * @param dc the deposit confirmation we know - */ -static void -test_dc (void *cls, - uint64_t serial_id, - const struct TALER_AUDITORDB_DepositConfirmation *dc) -{ - struct DepositConfirmationContext *dcc = cls; - enum GNUNET_DB_QueryStatus qs; - struct TALER_EXCHANGEDB_Deposit dep; - - dcc->last_seen_coin_serial = serial_id; - memset (&dep, - 0, - sizeof (dep)); - dep.coin.coin_pub = dc->coin_pub; - dep.h_contract_terms = dc->h_contract_terms; - dep.merchant_pub = dc->merchant; - dep.h_wire = dc->h_wire; - dep.refund_deadline = dc->refund_deadline; - - qs = edb->have_deposit (edb->cls, - esession, - &dep, - GNUNET_NO /* do not check refund deadline */); - if (qs > 0) - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Found deposit %s in exchange database\n", - GNUNET_h2s (&dc->h_contract_terms)); - return; /* found, all good */ - } - if (qs < 0) - { - GNUNET_break (0); /* DB error, complain */ - dcc->qs = qs; - return; - } - /* deposit confirmation missing! report! */ - report (report_deposit_confirmation_inconsistencies, - json_pack ("{s:o, s:o, s:I, s:o}", - "timestamp", - json_from_time_abs (dc->timestamp), - "amount", - TALER_JSON_from_amount (&dc->amount_without_fee), - "rowid", - (json_int_t) serial_id, - "account", - GNUNET_JSON_from_data_auto (&dc->h_wire))); - dcc->first_missed_coin_serial = GNUNET_MIN (dcc->first_missed_coin_serial, - serial_id); - dcc->missed_count++; - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&dcc->missed_amount, - &dcc->missed_amount, - &dc->amount_without_fee)); -} - - -/** - * Check that the deposit-confirmations that were reported to - * us by merchants are also in the exchange's database. - * - * @param cls closure - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -analyze_deposit_confirmations (void *cls) -{ - struct TALER_AUDITORDB_ProgressPointDepositConfirmation ppdc; - struct DepositConfirmationContext dcc; - enum GNUNET_DB_QueryStatus qs; - enum GNUNET_DB_QueryStatus qsx; - enum GNUNET_DB_QueryStatus qsp; - - (void) cls; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing deposit confirmations\n"); - ppdc.last_deposit_confirmation_serial_id = 0; - qsp = adb->get_auditor_progress_deposit_confirmation (adb->cls, - asession, - &master_pub, - &ppdc); - if (0 > qsp) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); - return qsp; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) - { - GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, - _ ( - "First analysis using this auditor, starting audit from scratch\n")); - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Resuming deposit confirmation audit at %llu\n"), - (unsigned long long) ppdc.last_deposit_confirmation_serial_id); - } - - /* setup 'cc' */ - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &dcc.missed_amount)); - dcc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - dcc.missed_count = 0LLU; - dcc.first_missed_coin_serial = UINT64_MAX; - qsx = adb->get_deposit_confirmations (adb->cls, - asession, - &master_pub, - ppdc.last_deposit_confirmation_serial_id, - &test_dc, - &dcc); - if (0 > qsx) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); - return qsx; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzed %d deposit confirmations (above serial ID %llu)\n", - (int) qsx, - (unsigned long long) ppdc.last_deposit_confirmation_serial_id); - if (0 > dcc.qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == dcc.qs); - return dcc.qs; - } - if (UINT64_MAX == dcc.first_missed_coin_serial) - ppdc.last_deposit_confirmation_serial_id = dcc.last_seen_coin_serial; - else - ppdc.last_deposit_confirmation_serial_id = dcc.first_missed_coin_serial - 1; - - /* sync 'cc' back to disk */ - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) - qs = adb->update_auditor_progress_deposit_confirmation (adb->cls, - asession, - &master_pub, - &ppdc); - else - qs = adb->insert_auditor_progress_deposit_confirmation (adb->cls, - asession, - &master_pub, - &ppdc); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to update auditor DB, not recording progress\n"); - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - number_missed_deposit_confirmations = (json_int_t) dcc.missed_count; - total_missed_deposit_confirmations = dcc.missed_amount; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Concluded deposit confirmation audit step at %llu\n"), - (unsigned long long) ppdc.last_deposit_confirmation_serial_id); - return qs; -} - - -/** - * Main function that will be run. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - json_t *report; - - (void) cls; - (void) args; - (void) cfgfile; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Launching auditor\n"); - if (GNUNET_OK != - setup_globals (c)) - { - global_ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting audit\n"); - GNUNET_assert (NULL != - (report_deposit_confirmation_inconsistencies = json_array ())); - if (GNUNET_OK != - setup_sessions_and_run (&analyze_deposit_confirmations, - NULL)) - { - global_ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Audit complete\n"); - report = json_pack ("{s:o, s:o, s:I}", - "deposit_confirmation_inconsistencies", - report_deposit_confirmation_inconsistencies, - "missing_deposit_confirmation_count", - (json_int_t) number_missed_deposit_confirmations, - "missing_deposit_confirmation_total", - TALER_JSON_from_amount ( - &total_missed_deposit_confirmations) - ); - GNUNET_break (NULL != report); - finish_report (report); -} - - -/** - * The main function of the database initialization tool. - * Used to initialize the Taler Exchange's database. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, - char *const *argv) -{ - const struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_base32_auto ('m', - "exchange-key", - "KEY", - "public key of the exchange (Crockford base32 encoded)", - &master_pub), - GNUNET_GETOPT_option_flag ('r', - "restart", - "restart audit from the beginning (required on first run)", - &restart), - GNUNET_GETOPT_option_timetravel ('T', - "timetravel"), - GNUNET_GETOPT_OPTION_END - }; - - /* force linker to link against libtalerutil; if we do - not do this, the linker may "optimize" libtalerutil - away and skip #TALER_OS_init(), which we do need */ - (void) TALER_project_data_default (); - GNUNET_assert (GNUNET_OK == - GNUNET_log_setup ("taler-auditor-deposits", - "MESSAGE", - NULL)); - if (GNUNET_OK != - GNUNET_PROGRAM_run (argc, - argv, - "taler-auditor-deposits", - "Audit Taler exchange database for deposit confirmation consistency", - options, - &run, - NULL)) - return 1; - return global_ret; -} - - -/* end of taler-auditor-deposits.c */ diff --git a/src/auditor/taler-auditor-reserves.c b/src/auditor/taler-auditor-reserves.c deleted file mode 100644 index 2fe103c81..000000000 --- a/src/auditor/taler-auditor-reserves.c +++ /dev/null @@ -1,1641 +0,0 @@ -/* - This file is part of TALER - Copyright (C) 2016-2020 Taler Systems SA - - TALER is free software; you can redistribute it and/or modify it under the - terms of the GNU Affero Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - TALER is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. - - You should have received a copy of the GNU Affero Public License along with - TALER; see the file COPYING. If not, see -*/ -/** - * @file auditor/taler-auditor-reserves.c - * @brief audits the reserves of an exchange database - * @author Christian Grothoff - */ -#include "platform.h" -#include -#include "taler_auditordb_plugin.h" -#include "taler_exchangedb_lib.h" -#include "taler_json_lib.h" -#include "taler_bank_service.h" -#include "taler_signatures.h" -#include "report-lib.h" - - -/** - * Use a 1 day grace period to deal with clocks not being perfectly synchronized. - */ -#define CLOSING_GRACE_PERIOD GNUNET_TIME_UNIT_DAYS - -/** - * Return value from main(). - */ -static int global_ret; - -/** - * After how long should idle reserves be closed? - */ -static struct GNUNET_TIME_Relative idle_reserve_expiration_time; - -/** - * Checkpointing our progress for reserves. - */ -static struct TALER_AUDITORDB_ProgressPointReserve ppr; - -/** - * Checkpointing our progress for reserves. - */ -static struct TALER_AUDITORDB_ProgressPointReserve ppr_start; - -/** - * Array of reports about row inconsitencies. - */ -static json_t *report_row_inconsistencies; - -/** - * Array of reports about the denomination key not being - * valid at the time of withdrawal. - */ -static json_t *denomination_key_validity_withdraw_inconsistencies; - -/** - * Array of reports about reserve balance insufficient inconsitencies. - */ -static json_t *report_reserve_balance_insufficient_inconsistencies; - -/** - * Total amount reserves were charged beyond their balance. - */ -static struct TALER_Amount total_balance_insufficient_loss; - -/** - * Array of reports about reserve balance summary wrong in database. - */ -static json_t *report_reserve_balance_summary_wrong_inconsistencies; - -/** - * Total delta between expected and stored reserve balance summaries, - * for positive deltas. - */ -static struct TALER_Amount total_balance_summary_delta_plus; - -/** - * Total delta between expected and stored reserve balance summaries, - * for negative deltas. - */ -static struct TALER_Amount total_balance_summary_delta_minus; - -/** - * Array of reports about reserve's not being closed inconsitencies. - */ -static json_t *report_reserve_not_closed_inconsistencies; - -/** - * Total amount affected by reserves not having been closed on time. - */ -static struct TALER_Amount total_balance_reserve_not_closed; - -/** - * Report about amount calculation differences (causing profit - * or loss at the exchange). - */ -static json_t *report_amount_arithmetic_inconsistencies; - -/** - * Profits the exchange made by bad amount calculations. - */ -static struct TALER_Amount total_arithmetic_delta_plus; - -/** - * Losses the exchange made by bad amount calculations. - */ -static struct TALER_Amount total_arithmetic_delta_minus; - -/** - * Expected balance in the escrow account. - */ -static struct TALER_Amount total_escrow_balance; - -/** - * Recoups we made on denominations that were not revoked (!?). - */ -static struct TALER_Amount total_irregular_recoups; - -/** - * Total withdraw fees earned. - */ -static struct TALER_Amount total_withdraw_fee_income; - -/** - * Array of reports about coin operations with bad signatures. - */ -static json_t *report_bad_sig_losses; - -/** - * Total amount lost by operations for which signatures were invalid. - */ -static struct TALER_Amount total_bad_sig_loss; - - -/* ***************************** Report logic **************************** */ - - -/** - * Report a (serious) inconsistency in the exchange's database with - * respect to calculations involving amounts. - * - * @param operation what operation had the inconsistency - * @param rowid affected row, UINT64_MAX if row is missing - * @param exchange amount calculated by exchange - * @param auditor amount calculated by auditor - * @param profitable 1 if @a exchange being larger than @a auditor is - * profitable for the exchange for this operation, - * -1 if @a exchange being smaller than @a auditor is - * profitable for the exchange, and 0 if it is unclear - */ -static void -report_amount_arithmetic_inconsistency (const char *operation, - uint64_t rowid, - const struct TALER_Amount *exchange, - const struct TALER_Amount *auditor, - int profitable) -{ - struct TALER_Amount delta; - struct TALER_Amount *target; - - if (0 < TALER_amount_cmp (exchange, - auditor)) - { - /* exchange > auditor */ - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - exchange, - auditor)); - } - else - { - /* auditor < exchange */ - profitable = -profitable; - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - auditor, - exchange)); - } - report (report_amount_arithmetic_inconsistencies, - json_pack ("{s:s, s:I, s:o, s:o, s:I}", - "operation", operation, - "rowid", (json_int_t) rowid, - "exchange", TALER_JSON_from_amount (exchange), - "auditor", TALER_JSON_from_amount (auditor), - "profitable", (json_int_t) profitable)); - if (0 != profitable) - { - target = (1 == profitable) - ? &total_arithmetic_delta_plus - : &total_arithmetic_delta_minus; - GNUNET_break (GNUNET_OK == - TALER_amount_add (target, - target, - &delta)); - } -} - - -/** - * Report a (serious) inconsistency in the exchange's database. - * - * @param table affected table - * @param rowid affected row, UINT64_MAX if row is missing - * @param diagnostic message explaining the problem - */ -static void -report_row_inconsistency (const char *table, - uint64_t rowid, - const char *diagnostic) -{ - report (report_row_inconsistencies, - json_pack ("{s:s, s:I, s:s}", - "table", table, - "row", (json_int_t) rowid, - "diagnostic", diagnostic)); -} - - -/* ***************************** Analyze reserves ************************ */ -/* This logic checks the reserves_in, reserves_out and reserves-tables */ - -/** - * Summary data we keep per reserve. - */ -struct ReserveSummary -{ - /** - * Public key of the reserve. - * Always set when the struct is first initialized. - */ - struct TALER_ReservePublicKeyP reserve_pub; - - /** - * Sum of all incoming transfers during this transaction. - * Updated only in #handle_reserve_in(). - */ - struct TALER_Amount total_in; - - /** - * Sum of all outgoing transfers during this transaction (includes fees). - * Updated only in #handle_reserve_out(). - */ - struct TALER_Amount total_out; - - /** - * Sum of withdraw fees encountered during this transaction. - */ - struct TALER_Amount total_fee; - - /** - * Previous balance of the reserve as remembered by the auditor. - * (updated based on @e total_in and @e total_out at the end). - */ - struct TALER_Amount a_balance; - - /** - * Previous withdraw fee balance of the reserve, as remembered by the auditor. - * (updated based on @e total_fee at the end). - */ - struct TALER_Amount a_withdraw_fee_balance; - - /** - * Previous reserve expiration data, as remembered by the auditor. - * (updated on-the-fly in #handle_reserve_in()). - */ - struct GNUNET_TIME_Absolute a_expiration_date; - - /** - * Which account did originally put money into the reserve? - */ - char *sender_account; - - /** - * Did we have a previous reserve info? Used to decide between - * UPDATE and INSERT later. Initialized in - * #load_auditor_reserve_summary() together with the a-* values - * (if available). - */ - int had_ri; - -}; - - -/** - * Load the auditor's remembered state about the reserve into @a rs. - * The "total_in" and "total_out" amounts of @a rs must already be - * initialized (so we can determine the currency). - * - * @param[in,out] rs reserve summary to (fully) initialize - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -load_auditor_reserve_summary (struct ReserveSummary *rs) -{ - enum GNUNET_DB_QueryStatus qs; - uint64_t rowid; - - qs = adb->get_reserve_info (adb->cls, - asession, - &rs->reserve_pub, - &master_pub, - &rowid, - &rs->a_balance, - &rs->a_withdraw_fee_balance, - &rs->a_expiration_date, - &rs->sender_account); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - rs->had_ri = GNUNET_NO; - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (rs->total_in.currency, - &rs->a_balance)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (rs->total_in.currency, - &rs->a_withdraw_fee_balance)); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Creating fresh reserve `%s' with starting balance %s\n", - TALER_B2S (&rs->reserve_pub), - TALER_amount2s (&rs->a_balance)); - return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; - } - rs->had_ri = GNUNET_YES; - if ( (GNUNET_YES != - TALER_amount_cmp_currency (&rs->a_balance, - &rs->a_withdraw_fee_balance)) || - (GNUNET_YES != - TALER_amount_cmp_currency (&rs->total_in, - &rs->a_balance)) ) - { - GNUNET_break (0); - return GNUNET_DB_STATUS_HARD_ERROR; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Auditor remembers reserve `%s' has balance %s\n", - TALER_B2S (&rs->reserve_pub), - TALER_amount2s (&rs->a_balance)); - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; -} - - -/** - * Closure to the various callbacks we make while checking a reserve. - */ -struct ReserveContext -{ - /** - * Map from hash of reserve's public key to a `struct ReserveSummary`. - */ - struct GNUNET_CONTAINER_MultiHashMap *reserves; - - /** - * Map from hash of denomination's public key to a - * static string "revoked" for keys that have been revoked, - * or "master signature invalid" in case the revocation is - * there but bogus. - */ - struct GNUNET_CONTAINER_MultiHashMap *revoked; - - /** - * Transaction status code, set to error codes if applicable. - */ - enum GNUNET_DB_QueryStatus qs; - -}; - - -/** - * Function called with details about incoming wire transfers. - * - * @param cls our `struct ReserveContext` - * @param rowid unique serial ID for the refresh session in our DB - * @param reserve_pub public key of the reserve (also the WTID) - * @param credit amount that was received - * @param sender_account_details information about the sender's bank account - * @param wire_reference unique reference identifying the wire transfer - * @param execution_date when did we receive the funds - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -handle_reserve_in (void *cls, - uint64_t rowid, - const struct TALER_ReservePublicKeyP *reserve_pub, - const struct TALER_Amount *credit, - const char *sender_account_details, - uint64_t wire_reference, - struct GNUNET_TIME_Absolute execution_date) -{ - struct ReserveContext *rc = cls; - struct GNUNET_HashCode key; - struct ReserveSummary *rs; - struct GNUNET_TIME_Absolute expiry; - enum GNUNET_DB_QueryStatus qs; - - (void) wire_reference; - /* should be monotonically increasing */ - GNUNET_assert (rowid >= ppr.last_reserve_in_serial_id); - ppr.last_reserve_in_serial_id = rowid + 1; - - GNUNET_CRYPTO_hash (reserve_pub, - sizeof (*reserve_pub), - &key); - rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, - &key); - if (NULL == rs) - { - rs = GNUNET_new (struct ReserveSummary); - rs->sender_account = GNUNET_strdup (sender_account_details); - rs->reserve_pub = *reserve_pub; - rs->total_in = *credit; - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (credit->currency, - &rs->total_out)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (credit->currency, - &rs->total_fee)); - if (0 > (qs = load_auditor_reserve_summary (rs))) - { - GNUNET_break (0); - GNUNET_free (rs); - rc->qs = qs; - return GNUNET_SYSERR; - } - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (rc->reserves, - &key, - rs, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - else - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&rs->total_in, - &rs->total_in, - credit)); - if (NULL == rs->sender_account) - rs->sender_account = GNUNET_strdup (sender_account_details); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Additional incoming wire transfer for reserve `%s' of %s\n", - TALER_B2S (reserve_pub), - TALER_amount2s (credit)); - expiry = GNUNET_TIME_absolute_add (execution_date, - idle_reserve_expiration_time); - rs->a_expiration_date = GNUNET_TIME_absolute_max (rs->a_expiration_date, - expiry); - return GNUNET_OK; -} - - -/** - * Function called with details about withdraw operations. Verifies - * the signature and updates the reserve's balance. - * - * @param cls our `struct ReserveContext` - * @param rowid unique serial ID for the refresh session in our DB - * @param h_blind_ev blinded hash of the coin's public key - * @param denom_pub public denomination key of the deposited coin - * @param reserve_pub public key of the reserve - * @param reserve_sig signature over the withdraw operation - * @param execution_date when did the wallet withdraw the coin - * @param amount_with_fee amount that was withdrawn - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -handle_reserve_out (void *cls, - uint64_t rowid, - const struct GNUNET_HashCode *h_blind_ev, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_ReservePublicKeyP *reserve_pub, - const struct TALER_ReserveSignatureP *reserve_sig, - struct GNUNET_TIME_Absolute execution_date, - const struct TALER_Amount *amount_with_fee) -{ - struct ReserveContext *rc = cls; - struct TALER_WithdrawRequestPS wsrd; - struct GNUNET_HashCode key; - struct ReserveSummary *rs; - const struct TALER_DenominationKeyValidityPS *issue; - struct TALER_Amount withdraw_fee; - struct GNUNET_TIME_Absolute valid_start; - struct GNUNET_TIME_Absolute expire_withdraw; - enum GNUNET_DB_QueryStatus qs; - - /* should be monotonically increasing */ - GNUNET_assert (rowid >= ppr.last_reserve_out_serial_id); - ppr.last_reserve_out_serial_id = rowid + 1; - - /* lookup denomination pub data (make sure denom_pub is valid, establish fees) */ - qs = get_denomination_info (denom_pub, - &issue, - &wsrd.h_denomination_pub); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - if (GNUNET_DB_STATUS_HARD_ERROR == qs) - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Hard database error trying to get denomination %s (%s) from database!\n", - TALER_B2S (denom_pub), - TALER_amount2s (amount_with_fee)); - rc->qs = qs; - return GNUNET_SYSERR; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("withdraw", - rowid, - "denomination key not found"); - return GNUNET_OK; - } - - /* check that execution date is within withdraw range for denom_pub */ - valid_start = GNUNET_TIME_absolute_ntoh (issue->start); - expire_withdraw = GNUNET_TIME_absolute_ntoh (issue->expire_withdraw); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Checking withdraw timing: %llu, expire: %llu, timing: %llu\n", - (unsigned long long) valid_start.abs_value_us, - (unsigned long long) expire_withdraw.abs_value_us, - (unsigned long long) execution_date.abs_value_us); - if ( (valid_start.abs_value_us > execution_date.abs_value_us) || - (expire_withdraw.abs_value_us < execution_date.abs_value_us) ) - { - report (denomination_key_validity_withdraw_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o}", - "row", (json_int_t) rowid, - "execution_date", - json_from_time_abs (execution_date), - "reserve_pub", GNUNET_JSON_from_data_auto (reserve_pub), - "denompub_h", GNUNET_JSON_from_data_auto ( - &wsrd.h_denomination_pub))); - } - - /* check reserve_sig */ - wsrd.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_RESERVE_WITHDRAW); - wsrd.purpose.size = htonl (sizeof (wsrd)); - wsrd.reserve_pub = *reserve_pub; - TALER_amount_hton (&wsrd.amount_with_fee, - amount_with_fee); - wsrd.withdraw_fee = issue->fee_withdraw; - wsrd.h_coin_envelope = *h_blind_ev; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_RESERVE_WITHDRAW, - &wsrd.purpose, - &reserve_sig->eddsa_signature, - &reserve_pub->eddsa_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "withdraw", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount_with_fee), - "key_pub", GNUNET_JSON_from_data_auto (reserve_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount_with_fee)); - return GNUNET_OK; - } - - GNUNET_CRYPTO_hash (reserve_pub, - sizeof (*reserve_pub), - &key); - rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, - &key); - if (NULL == rs) - { - rs = GNUNET_new (struct ReserveSummary); - rs->reserve_pub = *reserve_pub; - rs->total_out = *amount_with_fee; - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount_with_fee->currency, - &rs->total_in)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount_with_fee->currency, - &rs->total_fee)); - qs = load_auditor_reserve_summary (rs); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - GNUNET_free (rs); - rc->qs = qs; - return GNUNET_SYSERR; - } - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (rc->reserves, - &key, - rs, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - else - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&rs->total_out, - &rs->total_out, - amount_with_fee)); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Reserve `%s' reduced by %s from withdraw\n", - TALER_B2S (reserve_pub), - TALER_amount2s (amount_with_fee)); - TALER_amount_ntoh (&withdraw_fee, - &issue->fee_withdraw); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Increasing withdraw profits by fee %s\n", - TALER_amount2s (&withdraw_fee)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&rs->total_fee, - &rs->total_fee, - &withdraw_fee)); - - return GNUNET_OK; -} - - -/** - * Function called with details about withdraw operations. Verifies - * the signature and updates the reserve's balance. - * - * @param cls our `struct ReserveContext` - * @param rowid unique serial ID for the refresh session in our DB - * @param timestamp when did we receive the recoup request - * @param amount how much should be added back to the reserve - * @param reserve_pub public key of the reserve - * @param coin public information about the coin, denomination signature is - * already verified in #check_recoup() - * @param denom_pub public key of the denomionation of @a coin - * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP - * @param coin_blind blinding factor used to blind the coin - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -handle_recoup_by_reserve (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute timestamp, - const struct TALER_Amount *amount, - const struct TALER_ReservePublicKeyP *reserve_pub, - const struct TALER_CoinPublicInfo *coin, - const struct TALER_DenominationPublicKey *denom_pub, - const struct TALER_CoinSpendSignatureP *coin_sig, - const struct - TALER_DenominationBlindingKeyP *coin_blind) -{ - struct ReserveContext *rc = cls; - struct GNUNET_HashCode key; - struct ReserveSummary *rs; - struct GNUNET_TIME_Absolute expiry; - struct TALER_RecoupRequestPS pr; - struct TALER_MasterSignatureP msig; - uint64_t rev_rowid; - enum GNUNET_DB_QueryStatus qs; - const char *rev; - - (void) denom_pub; - /* should be monotonically increasing */ - GNUNET_assert (rowid >= ppr.last_reserve_recoup_serial_id); - ppr.last_reserve_recoup_serial_id = rowid + 1; - /* We know that denom_pub matches denom_pub_hash because this - is how the SQL statement joined the tables. */ - pr.h_denom_pub = coin->denom_pub_hash; - pr.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_RECOUP); - pr.purpose.size = htonl (sizeof (pr)); - pr.coin_pub = coin->coin_pub; - pr.coin_blind = *coin_blind; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_RECOUP, - &pr.purpose, - &coin_sig->eddsa_signature, - &coin->coin_pub.eddsa_pub)) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "recoup", - "row", (json_int_t) rowid, - "loss", TALER_JSON_from_amount (amount), - "key_pub", GNUNET_JSON_from_data_auto ( - &coin->coin_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount)); - } - - /* check that the coin was eligible for recoup!*/ - rev = GNUNET_CONTAINER_multihashmap_get (rc->revoked, - &pr.h_denom_pub); - if (NULL == rev) - { - qs = edb->get_denomination_revocation (edb->cls, - esession, - &pr.h_denom_pub, - &msig, - &rev_rowid); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - rc->qs = qs; - return GNUNET_SYSERR; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - report_row_inconsistency ("recoup", - rowid, - "denomination key not in revocation set"); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_irregular_recoups, - &total_irregular_recoups, - amount)); - } - else - { - /* verify msig */ - struct TALER_MasterDenominationKeyRevocationPS kr; - - kr.purpose.purpose = htonl ( - TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED); - kr.purpose.size = htonl (sizeof (kr)); - kr.h_denom_pub = pr.h_denom_pub; - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_verify ( - TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED, - &kr.purpose, - &msig.eddsa_signature, - &master_pub.eddsa_pub)) - { - rev = "master signature invalid"; - } - else - { - rev = "revoked"; - } - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (rc->revoked, - &pr.h_denom_pub, - (void *) rev, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - } - else - { - rev_rowid = 0; /* reported elsewhere */ - } - if ( (NULL != rev) && - (0 == strcmp (rev, "master signature invalid")) ) - { - report (report_bad_sig_losses, - json_pack ("{s:s, s:I, s:o, s:o}", - "operation", "recoup-master", - "row", (json_int_t) rev_rowid, - "loss", TALER_JSON_from_amount (amount), - "key_pub", GNUNET_JSON_from_data_auto (&master_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_sig_loss, - &total_bad_sig_loss, - amount)); - } - - GNUNET_CRYPTO_hash (reserve_pub, - sizeof (*reserve_pub), - &key); - rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, - &key); - if (NULL == rs) - { - rs = GNUNET_new (struct ReserveSummary); - rs->reserve_pub = *reserve_pub; - rs->total_in = *amount; - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount->currency, - &rs->total_out)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount->currency, - &rs->total_fee)); - qs = load_auditor_reserve_summary (rs); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - GNUNET_free (rs); - rc->qs = qs; - return GNUNET_SYSERR; - } - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (rc->reserves, - &key, - rs, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - else - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&rs->total_in, - &rs->total_in, - amount)); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Additional /recoup value to for reserve `%s' of %s\n", - TALER_B2S (reserve_pub), - TALER_amount2s (amount)); - expiry = GNUNET_TIME_absolute_add (timestamp, - idle_reserve_expiration_time); - rs->a_expiration_date = GNUNET_TIME_absolute_max (rs->a_expiration_date, - expiry); - return GNUNET_OK; -} - - -/** - * Obtain the closing fee for a transfer at @a time for target - * @a receiver_account. - * - * @param receiver_account payto:// URI of the target account - * @param atime when was the transfer made - * @param[out] fee set to the closing fee - * @return #GNUNET_OK on success - */ -static int -get_closing_fee (const char *receiver_account, - struct GNUNET_TIME_Absolute atime, - struct TALER_Amount *fee) -{ - struct TALER_MasterSignatureP master_sig; - struct GNUNET_TIME_Absolute start_date; - struct GNUNET_TIME_Absolute end_date; - struct TALER_Amount wire_fee; - char *method; - - method = TALER_payto_get_method (receiver_account); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Method is `%s'\n", - method); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != - edb->get_wire_fee (edb->cls, - esession, - method, - atime, - &start_date, - &end_date, - &wire_fee, - fee, - &master_sig)) - { - report_row_inconsistency ("closing-fee", - atime.abs_value_us, - "closing fee unavailable at given time"); - GNUNET_free (method); - return GNUNET_SYSERR; - } - GNUNET_free (method); - return GNUNET_OK; -} - - -/** - * Function called about reserve closing operations - * the aggregator triggered. - * - * @param cls closure - * @param rowid row identifier used to uniquely identify the reserve closing operation - * @param execution_date when did we execute the close operation - * @param amount_with_fee how much did we debit the reserve - * @param closing_fee how much did we charge for closing the reserve - * @param reserve_pub public key of the reserve - * @param receiver_account where did we send the funds - * @param transfer_details details about the wire transfer - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -handle_reserve_closed (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute execution_date, - const struct TALER_Amount *amount_with_fee, - const struct TALER_Amount *closing_fee, - const struct TALER_ReservePublicKeyP *reserve_pub, - const char *receiver_account, - const struct - TALER_WireTransferIdentifierRawP *transfer_details) -{ - struct ReserveContext *rc = cls; - struct GNUNET_HashCode key; - struct ReserveSummary *rs; - enum GNUNET_DB_QueryStatus qs; - - (void) transfer_details; - /* should be monotonically increasing */ - GNUNET_assert (rowid >= ppr.last_reserve_close_serial_id); - ppr.last_reserve_close_serial_id = rowid + 1; - - GNUNET_CRYPTO_hash (reserve_pub, - sizeof (*reserve_pub), - &key); - rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, - &key); - if (NULL == rs) - { - rs = GNUNET_new (struct ReserveSummary); - rs->reserve_pub = *reserve_pub; - rs->total_out = *amount_with_fee; - rs->total_fee = *closing_fee; - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (amount_with_fee->currency, - &rs->total_in)); - qs = load_auditor_reserve_summary (rs); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - GNUNET_free (rs); - rc->qs = qs; - return GNUNET_SYSERR; - } - GNUNET_assert (GNUNET_OK == - GNUNET_CONTAINER_multihashmap_put (rc->reserves, - &key, - rs, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); - } - else - { - struct TALER_Amount expected_fee; - - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&rs->total_out, - &rs->total_out, - amount_with_fee)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&rs->total_fee, - &rs->total_fee, - closing_fee)); - /* verify closing_fee is correct! */ - if (GNUNET_OK != - get_closing_fee (receiver_account, - execution_date, - &expected_fee)) - { - GNUNET_break (0); - } - else if (0 != TALER_amount_cmp (&expected_fee, - closing_fee)) - { - report_amount_arithmetic_inconsistency ("closing aggregation fee", - rowid, - closing_fee, - &expected_fee, - 1); - } - } - if (NULL == rs->sender_account) - { - GNUNET_break (GNUNET_NO == rs->had_ri); - report_row_inconsistency ("reserves_close", - rowid, - "target account not verified, auditor does not know reserve"); - } - else if (0 != strcmp (rs->sender_account, - receiver_account)) - { - report_row_inconsistency ("reserves_close", - rowid, - "target account does not match origin account"); - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Additional closing operation for reserve `%s' of %s\n", - TALER_B2S (reserve_pub), - TALER_amount2s (amount_with_fee)); - return GNUNET_OK; -} - - -/** - * Check that the reserve summary matches what the exchange database - * thinks about the reserve, and update our own state of the reserve. - * - * Remove all reserves that we are happy with from the DB. - * - * @param cls our `struct ReserveContext` - * @param key hash of the reserve public key - * @param value a `struct ReserveSummary` - * @return #GNUNET_OK to process more entries - */ -static int -verify_reserve_balance (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ReserveContext *rc = cls; - struct ReserveSummary *rs = value; - struct TALER_EXCHANGEDB_Reserve reserve; - struct TALER_Amount balance; - struct TALER_Amount nbalance; - struct TALER_Amount cfee; - enum GNUNET_DB_QueryStatus qs; - int ret; - - ret = GNUNET_OK; - reserve.pub = rs->reserve_pub; - qs = edb->reserves_get (edb->cls, - esession, - &reserve); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) - { - char *diag; - - GNUNET_asprintf (&diag, - "Failed to find summary for reserve `%s'\n", - TALER_B2S (&rs->reserve_pub)); - report_row_inconsistency ("reserve-summary", - UINT64_MAX, - diag); - GNUNET_free (diag); - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) - { - GNUNET_break (0); - qs = GNUNET_DB_STATUS_HARD_ERROR; - } - rc->qs = qs; - return GNUNET_OK; - } - - if (GNUNET_OK != - TALER_amount_add (&balance, - &rs->total_in, - &rs->a_balance)) - { - GNUNET_break (0); - goto cleanup; - } - - if (GNUNET_SYSERR == - TALER_amount_subtract (&nbalance, - &balance, - &rs->total_out)) - { - struct TALER_Amount loss; - - GNUNET_break (GNUNET_SYSERR != - TALER_amount_subtract (&loss, - &rs->total_out, - &balance)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_balance_insufficient_loss, - &total_balance_insufficient_loss, - &loss)); - report (report_reserve_balance_insufficient_inconsistencies, - json_pack ("{s:o, s:o}", - "reserve_pub", - GNUNET_JSON_from_data_auto (&rs->reserve_pub), - "loss", - TALER_JSON_from_amount (&loss))); - goto cleanup; - } - if (0 != TALER_amount_cmp (&nbalance, - &reserve.balance)) - { - struct TALER_Amount delta; - - if (0 < TALER_amount_cmp (&nbalance, - &reserve.balance)) - { - /* balance > reserve.balance */ - GNUNET_assert (GNUNET_OK == - TALER_amount_subtract (&delta, - &nbalance, - &reserve.balance)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&total_balance_summary_delta_plus, - &total_balance_summary_delta_plus, - &delta)); - } - else - { - /* balance < reserve.balance */ - GNUNET_assert (GNUNET_OK == - TALER_amount_subtract (&delta, - &reserve.balance, - &nbalance)); - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&total_balance_summary_delta_minus, - &total_balance_summary_delta_minus, - &delta)); - } - report (report_reserve_balance_summary_wrong_inconsistencies, - json_pack ("{s:o, s:o, s:o}", - "reserve_pub", - GNUNET_JSON_from_data_auto (&rs->reserve_pub), - "exchange", - TALER_JSON_from_amount (&reserve.balance), - "auditor", - TALER_JSON_from_amount (&nbalance))); - goto cleanup; - } - - /* Check that reserve is being closed if it is past its expiration date */ - - if (CLOSING_GRACE_PERIOD.rel_value_us < - GNUNET_TIME_absolute_get_duration (rs->a_expiration_date).rel_value_us) - { - if ( (NULL != rs->sender_account) && - (GNUNET_OK == - get_closing_fee (rs->sender_account, - rs->a_expiration_date, - &cfee)) ) - { - if (1 == TALER_amount_cmp (&nbalance, - &cfee)) - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&total_balance_reserve_not_closed, - &total_balance_reserve_not_closed, - &nbalance)); - report (report_reserve_not_closed_inconsistencies, - json_pack ("{s:o, s:o, s:o}", - "reserve_pub", - GNUNET_JSON_from_data_auto (&rs->reserve_pub), - "balance", - TALER_JSON_from_amount (&nbalance), - "expiration_time", - json_from_time_abs (rs->a_expiration_date))); - } - } - else - { - GNUNET_assert (GNUNET_OK == - TALER_amount_add (&total_balance_reserve_not_closed, - &total_balance_reserve_not_closed, - &nbalance)); - report (report_reserve_not_closed_inconsistencies, - json_pack ("{s:o, s:o, s:o, s:s}", - "reserve_pub", - GNUNET_JSON_from_data_auto (&rs->reserve_pub), - "balance", - TALER_JSON_from_amount (&nbalance), - "expiration_time", - json_from_time_abs (rs->a_expiration_date), - "diagnostic", - "could not determine closing fee")); - } - } - - /* Add withdraw fees we encountered to totals */ - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Reserve reserve `%s' made %s in withdraw fees\n", - TALER_B2S (&rs->reserve_pub), - TALER_amount2s (&rs->total_fee)); - if (GNUNET_YES != - TALER_amount_add (&rs->a_withdraw_fee_balance, - &rs->a_withdraw_fee_balance, - &rs->total_fee)) - { - GNUNET_break (0); - ret = GNUNET_SYSERR; - goto cleanup; - } - if ( (GNUNET_YES != - TALER_amount_add (&total_escrow_balance, - &total_escrow_balance, - &rs->total_in)) || - (GNUNET_SYSERR == - TALER_amount_subtract (&total_escrow_balance, - &total_escrow_balance, - &rs->total_out)) || - (GNUNET_YES != - TALER_amount_add (&total_withdraw_fee_income, - &total_withdraw_fee_income, - &rs->total_fee)) ) - { - GNUNET_break (0); - ret = GNUNET_SYSERR; - goto cleanup; - } - - if ( (0ULL == balance.value) && - (0U == balance.fraction) ) - { - /* balance is zero, drop reserve details (and then do not update/insert) */ - if (rs->had_ri) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Final balance of reserve `%s' is %s, dropping it\n", - TALER_B2S (&rs->reserve_pub), - TALER_amount2s (&nbalance)); - qs = adb->del_reserve_info (adb->cls, - asession, - &rs->reserve_pub, - &master_pub); - if (0 >= qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - ret = GNUNET_SYSERR; - rc->qs = qs; - goto cleanup; - } - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Final balance of reserve `%s' is %s, no need to remember it\n", - TALER_B2S (&rs->reserve_pub), - TALER_amount2s (&nbalance)); - } - ret = GNUNET_OK; - goto cleanup; - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Remembering final balance of reserve `%s' as %s\n", - TALER_B2S (&rs->reserve_pub), - TALER_amount2s (&nbalance)); - - if (rs->had_ri) - qs = adb->update_reserve_info (adb->cls, - asession, - &rs->reserve_pub, - &master_pub, - &nbalance, - &rs->a_withdraw_fee_balance, - rs->a_expiration_date); - else - qs = adb->insert_reserve_info (adb->cls, - asession, - &rs->reserve_pub, - &master_pub, - &nbalance, - &rs->a_withdraw_fee_balance, - rs->a_expiration_date, - rs->sender_account); - if (0 >= qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - ret = GNUNET_SYSERR; - rc->qs = qs; - } -cleanup: - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (rc->reserves, - key, - rs)); - GNUNET_free_non_null (rs->sender_account); - GNUNET_free (rs); - return ret; -} - - -/** - * Analyze reserves for being well-formed. - * - * @param cls NULL - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -analyze_reserves (void *cls) -{ - struct ReserveContext rc; - enum GNUNET_DB_QueryStatus qsx; - enum GNUNET_DB_QueryStatus qs; - enum GNUNET_DB_QueryStatus qsp; - - (void) cls; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Analyzing reserves\n"); - qsp = adb->get_auditor_progress_reserve (adb->cls, - asession, - &master_pub, - &ppr); - if (0 > qsp) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); - return qsp; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) - { - GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, - _ ( - "First analysis using this auditor, starting audit from scratch\n")); - } - else - { - ppr_start = ppr; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Resuming reserve audit at %llu/%llu/%llu/%llu\n"), - (unsigned long long) ppr.last_reserve_in_serial_id, - (unsigned long long) ppr.last_reserve_out_serial_id, - (unsigned long long) ppr.last_reserve_recoup_serial_id, - (unsigned long long) ppr.last_reserve_close_serial_id); - } - rc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; - qsx = adb->get_reserve_summary (adb->cls, - asession, - &master_pub, - &total_escrow_balance, - &total_withdraw_fee_income); - if (qsx < 0) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); - return qsx; - } - rc.reserves = GNUNET_CONTAINER_multihashmap_create (512, - GNUNET_NO); - rc.revoked = GNUNET_CONTAINER_multihashmap_create (4, - GNUNET_NO); - - qs = edb->select_reserves_in_above_serial_id (edb->cls, - esession, - ppr.last_reserve_in_serial_id, - &handle_reserve_in, - &rc); - if (qs < 0) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - qs = edb->select_withdrawals_above_serial_id (edb->cls, - esession, - ppr.last_reserve_out_serial_id, - &handle_reserve_out, - &rc); - if (qs < 0) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - qs = edb->select_recoup_above_serial_id (edb->cls, - esession, - ppr.last_reserve_recoup_serial_id, - &handle_recoup_by_reserve, - &rc); - if (qs < 0) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - qs = edb->select_reserve_closed_above_serial_id (edb->cls, - esession, - ppr. - last_reserve_close_serial_id, - &handle_reserve_closed, - &rc); - if (qs < 0) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - - GNUNET_CONTAINER_multihashmap_iterate (rc.reserves, - &verify_reserve_balance, - &rc); - GNUNET_break (0 == - GNUNET_CONTAINER_multihashmap_size (rc.reserves)); - GNUNET_CONTAINER_multihashmap_destroy (rc.reserves); - GNUNET_CONTAINER_multihashmap_destroy (rc.revoked); - - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != rc.qs) - return qs; - - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsx) - { - qs = adb->insert_reserve_summary (adb->cls, - asession, - &master_pub, - &total_escrow_balance, - &total_withdraw_fee_income); - } - else - { - qs = adb->update_reserve_summary (adb->cls, - asession, - &master_pub, - &total_escrow_balance, - &total_withdraw_fee_income); - } - if (0 >= qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) - qs = adb->update_auditor_progress_reserve (adb->cls, - asession, - &master_pub, - &ppr); - else - qs = adb->insert_auditor_progress_reserve (adb->cls, - asession, - &master_pub, - &ppr); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to update auditor DB, not recording progress\n"); - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _ ("Concluded reserve audit step at %llu/%llu/%llu/%llu\n"), - (unsigned long long) ppr.last_reserve_in_serial_id, - (unsigned long long) ppr.last_reserve_out_serial_id, - (unsigned long long) ppr.last_reserve_recoup_serial_id, - (unsigned long long) ppr.last_reserve_close_serial_id); - return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; -} - - -/** - * Main function that will be run. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - json_t *report; - - (void) cls; - (void) args; - (void) cfgfile; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Launching auditor\n"); - if (GNUNET_OK != - setup_globals (cfg)) - { - global_ret = 1; - return; - } - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_time (cfg, - "exchangedb", - "IDLE_RESERVE_EXPIRATION_TIME", - &idle_reserve_expiration_time)) - { - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, - "exchangedb", - "IDLE_RESERVE_EXPIRATION_TIME"); - global_ret = 1; - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting audit\n"); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_escrow_balance)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_irregular_recoups)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_withdraw_fee_income)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_balance_insufficient_loss)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_balance_summary_delta_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_balance_summary_delta_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_arithmetic_delta_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_arithmetic_delta_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_balance_reserve_not_closed)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_sig_loss)); - GNUNET_assert (NULL != - (report_row_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (denomination_key_validity_withdraw_inconsistencies = - json_array ())); - GNUNET_assert (NULL != - (report_reserve_balance_summary_wrong_inconsistencies = - json_array ())); - GNUNET_assert (NULL != - (report_reserve_balance_insufficient_inconsistencies = - json_array ())); - GNUNET_assert (NULL != - (report_reserve_not_closed_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_amount_arithmetic_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_bad_sig_losses = json_array ())); - if (GNUNET_OK != - setup_sessions_and_run (&analyze_reserves, - NULL)) - { - global_ret = 1; - return; - } - report = json_pack ("{s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:I," - " s:I, s:I, s:I, s:I, s:I," - " s:I, s:I }", - /* blocks #1 */ - "reserve_balance_insufficient_inconsistencies", - report_reserve_balance_insufficient_inconsistencies, - /* Tested in test-auditor.sh #3 */ - "total_loss_balance_insufficient", - TALER_JSON_from_amount (&total_balance_insufficient_loss), - /* Tested in test-auditor.sh #3 */ - "reserve_balance_summary_wrong_inconsistencies", - report_reserve_balance_summary_wrong_inconsistencies, - "total_balance_summary_delta_plus", - TALER_JSON_from_amount ( - &total_balance_summary_delta_plus), - "total_balance_summary_delta_minus", - TALER_JSON_from_amount ( - &total_balance_summary_delta_minus), - /* blocks #2 */ - "total_escrow_balance", - TALER_JSON_from_amount (&total_escrow_balance), - "total_withdraw_fee_income", - TALER_JSON_from_amount (&total_withdraw_fee_income), - /* Tested in test-auditor.sh #21 */ - "reserve_not_closed_inconsistencies", - report_reserve_not_closed_inconsistencies, - /* Tested in test-auditor.sh #21 */ - "total_balance_reserve_not_closed", - TALER_JSON_from_amount ( - &total_balance_reserve_not_closed), - /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ - "bad_sig_losses", - report_bad_sig_losses, - /* blocks #3 */ - /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ - "total_bad_sig_loss", - TALER_JSON_from_amount (&total_bad_sig_loss), - /* Tested in test-auditor.sh #14/#15 */ - "row_inconsistencies", - report_row_inconsistencies, - /* Tested in test-auditor.sh #23 */ - "denomination_key_validity_withdraw_inconsistencies", - denomination_key_validity_withdraw_inconsistencies, - "amount_arithmetic_inconsistencies", - report_amount_arithmetic_inconsistencies, - "total_arithmetic_delta_plus", - TALER_JSON_from_amount (&total_arithmetic_delta_plus), - /* blocks #4 */ - "total_arithmetic_delta_minus", - TALER_JSON_from_amount (&total_arithmetic_delta_minus), - "auditor_start_time", - json_from_time_abs (start_time), - "auditor_end_time", - json_from_time_abs (GNUNET_TIME_absolute_get ()), - "total_irregular_recoups", - TALER_JSON_from_amount (&total_irregular_recoups), - "start_ppr_reserve_in_serial_id", - (json_int_t) ppr_start.last_reserve_in_serial_id, - /* blocks #5 */ - "start_ppr_reserve_out_serial_id", - (json_int_t) ppr_start.last_reserve_out_serial_id, - "start_ppr_reserve_recoup_serial_id", - (json_int_t) ppr_start.last_reserve_recoup_serial_id, - "start_ppr_reserve_close_serial_id", - (json_int_t) ppr_start.last_reserve_close_serial_id, - "end_ppr_reserve_in_serial_id", - (json_int_t) ppr.last_reserve_in_serial_id, - "end_ppr_reserve_out_serial_id", - (json_int_t) ppr.last_reserve_out_serial_id, - /* blocks #6 */ - "end_ppr_reserve_recoup_serial_id", - (json_int_t) ppr.last_reserve_recoup_serial_id, - "end_ppr_reserve_close_serial_id", - (json_int_t) ppr.last_reserve_close_serial_id - ); - GNUNET_break (NULL != report); - finish_report (report); -} - - -/** - * The main function of the database initialization tool. - * Used to initialize the Taler Exchange's database. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, - char *const *argv) -{ - const struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_base32_auto ('m', - "exchange-key", - "KEY", - "public key of the exchange (Crockford base32 encoded)", - &master_pub), - GNUNET_GETOPT_option_flag ('r', - "restart", - "restart audit from the beginning (required on first run)", - &restart), - GNUNET_GETOPT_option_timetravel ('T', - "timetravel"), - GNUNET_GETOPT_OPTION_END - }; - - /* force linker to link against libtalerutil; if we do - not do this, the linker may "optimize" libtalerutil - away and skip #TALER_OS_init(), which we do need */ - (void) TALER_project_data_default (); - GNUNET_assert (GNUNET_OK == - GNUNET_log_setup ("taler-auditor-reserves", - "MESSAGE", - NULL)); - if (GNUNET_OK != - GNUNET_PROGRAM_run (argc, - argv, - "taler-auditor-reserves", - "Audit Taler exchange reserve handling", - options, - &run, - NULL)) - return 1; - return global_ret; -} - - -/* end of taler-auditor-reserves.c */ diff --git a/src/auditor/taler-helper-auditor-aggregation.c b/src/auditor/taler-helper-auditor-aggregation.c new file mode 100644 index 000000000..bbad71252 --- /dev/null +++ b/src/auditor/taler-helper-auditor-aggregation.c @@ -0,0 +1,1548 @@ +/* + This file is part of TALER + Copyright (C) 2016-2020 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. + + You should have received a copy of the GNU Affero Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file auditor/taler-helper-auditor-aggregation.c + * @brief audits an exchange's aggregations. + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include "taler_auditordb_plugin.h" +#include "taler_exchangedb_lib.h" +#include "taler_json_lib.h" +#include "taler_bank_service.h" +#include "taler_signatures.h" +#include "report-lib.h" + + +/** + * Return value from main(). + */ +static int global_ret; + +/** + * Checkpointing our progress for aggregations. + */ +static struct TALER_AUDITORDB_ProgressPointAggregation ppa; + +/** + * Checkpointing our progress for aggregations. + */ +static struct TALER_AUDITORDB_ProgressPointAggregation ppa_start; + +/** + * Array of reports about row inconsitencies. + */ +static json_t *report_row_inconsistencies; + +/** + * Array of reports about irregular wire out entries. + */ +static json_t *report_wire_out_inconsistencies; + +/** + * Total delta between calculated and stored wire out transfers, + * for positive deltas. + */ +static struct TALER_Amount total_wire_out_delta_plus; + +/** + * Total delta between calculated and stored wire out transfers + * for negative deltas. + */ +static struct TALER_Amount total_wire_out_delta_minus; + +/** + * Array of reports about inconsistencies about coins. + */ +static json_t *report_coin_inconsistencies; + +/** + * Profits the exchange made by bad amount calculations on coins. + */ +static struct TALER_Amount total_coin_delta_plus; + +/** + * Losses the exchange made by bad amount calculations on coins. + */ +static struct TALER_Amount total_coin_delta_minus; + +/** + * Report about amount calculation differences (causing profit + * or loss at the exchange). + */ +static json_t *report_amount_arithmetic_inconsistencies; + +/** + * Array of reports about wire fees being ambiguous in terms of validity periods. + */ +static json_t *report_fee_time_inconsistencies; + +/** + * Profits the exchange made by bad amount calculations. + */ +static struct TALER_Amount total_arithmetic_delta_plus; + +/** + * Losses the exchange made by bad amount calculations. + */ +static struct TALER_Amount total_arithmetic_delta_minus; + +/** + * Total aggregation fees earned. + */ +static struct TALER_Amount total_aggregation_fee_income; + +/** + * Array of reports about coin operations with bad signatures. + */ +static json_t *report_bad_sig_losses; + +/** + * Total amount lost by operations for which signatures were invalid. + */ +static struct TALER_Amount total_bad_sig_loss; + + +/** + * Report a (serious) inconsistency in the exchange's database with + * respect to calculations involving amounts. + * + * @param operation what operation had the inconsistency + * @param rowid affected row, UINT64_MAX if row is missing + * @param exchange amount calculated by exchange + * @param auditor amount calculated by auditor + * @param profitable 1 if @a exchange being larger than @a auditor is + * profitable for the exchange for this operation, + * -1 if @a exchange being smaller than @a auditor is + * profitable for the exchange, and 0 if it is unclear + */ +static void +report_amount_arithmetic_inconsistency (const char *operation, + uint64_t rowid, + const struct + TALER_Amount *exchange, + const struct + TALER_Amount *auditor, + int profitable) +{ + struct TALER_Amount delta; + struct TALER_Amount *target; + + if (0 < TALER_amount_cmp (exchange, + auditor)) + { + /* exchange > auditor */ + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + exchange, + auditor)); + } + else + { + /* auditor < exchange */ + profitable = -profitable; + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + auditor, + exchange)); + } + TALER_ARL_report (report_amount_arithmetic_inconsistencies, + json_pack ("{s:s, s:I, s:o, s:o, s:I}", + "operation", operation, + "rowid", (json_int_t) rowid, + "exchange", TALER_JSON_from_amount (exchange), + "auditor", TALER_JSON_from_amount (auditor), + "profitable", (json_int_t) profitable)); + if (0 != profitable) + { + target = (1 == profitable) + ? &total_arithmetic_delta_plus + : &total_arithmetic_delta_minus; + GNUNET_break (GNUNET_OK == + TALER_amount_add (target, + target, + &delta)); + } +} + + +/** + * Report a (serious) inconsistency in the exchange's database with + * respect to calculations involving amounts of a coin. + * + * @param operation what operation had the inconsistency + * @param coin_pub affected coin + * @param exchange amount calculated by exchange + * @param auditor amount calculated by auditor + * @param profitable 1 if @a exchange being larger than @a auditor is + * profitable for the exchange for this operation, + * -1 if @a exchange being smaller than @a auditor is + * profitable for the exchange, and 0 if it is unclear + */ +static void +report_coin_arithmetic_inconsistency (const char *operation, + const struct + TALER_CoinSpendPublicKeyP * + coin_pub, + const struct + TALER_Amount *exchange, + const struct + TALER_Amount *auditor, + int profitable) +{ + struct TALER_Amount delta; + struct TALER_Amount *target; + + if (0 < TALER_amount_cmp (exchange, + auditor)) + { + /* exchange > auditor */ + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + exchange, + auditor)); + } + else + { + /* auditor < exchange */ + profitable = -profitable; + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + auditor, + exchange)); + } + TALER_ARL_report (report_coin_inconsistencies, + json_pack ("{s:s, s:o, s:o, s:o, s:I}", + "operation", operation, + "coin_pub", GNUNET_JSON_from_data_auto ( + coin_pub), + "exchange", TALER_JSON_from_amount (exchange), + "auditor", TALER_JSON_from_amount (auditor), + "profitable", (json_int_t) profitable)); + if (0 != profitable) + { + target = (1 == profitable) + ? &total_coin_delta_plus + : &total_coin_delta_minus; + GNUNET_break (GNUNET_OK == + TALER_amount_add (target, + target, + &delta)); + } +} + + +/** + * Report a (serious) inconsistency in the exchange's database. + * + * @param table affected table + * @param rowid affected row, UINT64_MAX if row is missing + * @param diagnostic message explaining the problem + */ +static void +report_row_inconsistency (const char *table, + uint64_t rowid, + const char *diagnostic) +{ + TALER_ARL_report (report_row_inconsistencies, + json_pack ("{s:s, s:I, s:s}", + "table", table, + "row", (json_int_t) rowid, + "diagnostic", diagnostic)); +} + + +/* *********************** Analyze aggregations ******************** */ +/* This logic checks that the aggregator did the right thing + paying each merchant what they were due (and on time). */ + + +/** + * Information about wire fees charged by the exchange. + */ +struct WireFeeInfo +{ + + /** + * Kept in a DLL. + */ + struct WireFeeInfo *next; + + /** + * Kept in a DLL. + */ + struct WireFeeInfo *prev; + + /** + * When does the fee go into effect (inclusive). + */ + struct GNUNET_TIME_Absolute start_date; + + /** + * When does the fee stop being in effect (exclusive). + */ + struct GNUNET_TIME_Absolute end_date; + + /** + * How high is the wire fee. + */ + struct TALER_Amount wire_fee; + + /** + * How high is the closing fee. + */ + struct TALER_Amount closing_fee; + +}; + + +/** + * Closure for callbacks during #analyze_merchants(). + */ +struct AggregationContext +{ + + /** + * DLL of wire fees charged by the exchange. + */ + struct WireFeeInfo *fee_head; + + /** + * DLL of wire fees charged by the exchange. + */ + struct WireFeeInfo *fee_tail; + + /** + * Final result status. + */ + enum GNUNET_DB_QueryStatus qs; +}; + + +/** + * Closure for #wire_transfer_information_cb. + */ +struct WireCheckContext +{ + + /** + * Corresponding merchant context. + */ + struct AggregationContext *ac; + + /** + * Total deposits claimed by all transactions that were aggregated + * under the given @e wtid. + */ + struct TALER_Amount total_deposits; + + /** + * Hash of the wire transfer details of the receiver. + */ + struct GNUNET_HashCode h_wire; + + /** + * Execution time of the wire transfer. + */ + struct GNUNET_TIME_Absolute date; + + /** + * Database transaction status. + */ + enum GNUNET_DB_QueryStatus qs; + +}; + + +/** + * Check coin's transaction history for plausibility. Does NOT check + * the signatures (those are checked independently), but does calculate + * the amounts for the aggregation table and checks that the total + * claimed coin value is within the value of the coin's denomination. + * + * @param coin_pub public key of the coin (for TALER_ARL_reporting) + * @param h_contract_terms hash of the proposal for which we calculate the amount + * @param merchant_pub public key of the merchant (who is allowed to issue refunds) + * @param issue denomination information about the coin + * @param tl_head head of transaction history to verify + * @param[out] merchant_gain amount the coin contributes to the wire transfer to the merchant + * @param[out] deposit_gain amount the coin contributes excluding refunds + * @return #GNUNET_OK on success, #GNUNET_SYSERR on error + */ +static int +check_transaction_history_for_deposit (const struct + TALER_CoinSpendPublicKeyP * + coin_pub, + const struct + GNUNET_HashCode * + h_contract_terms, + const struct + TALER_MerchantPublicKeyP * + merchant_pub, + const struct + TALER_DenominationKeyValidityPS + *issue, + const struct + TALER_EXCHANGEDB_TransactionList + *tl_head, + struct TALER_Amount * + merchant_gain, + struct TALER_Amount * + deposit_gain) +{ + struct TALER_Amount expenditures; + struct TALER_Amount refunds; + struct TALER_Amount spent; + struct TALER_Amount value; + struct TALER_Amount merchant_loss; + struct TALER_Amount final_gain; + const struct TALER_Amount *deposit_fee; + int refund_deposit_fee; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Checking transaction history of coin %s\n", + TALER_B2S (coin_pub)); + + GNUNET_assert (NULL != tl_head); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &expenditures)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &refunds)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + merchant_gain)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &merchant_loss)); + /* Go over transaction history to compute totals; note that we do not + know the order, so instead of subtracting we compute positive + (deposit, melt) and negative (refund) values separately here, + and then subtract the negative from the positive after the loop. */ + refund_deposit_fee = GNUNET_NO; + deposit_fee = NULL; + for (const struct TALER_EXCHANGEDB_TransactionList *tl = tl_head; + NULL != tl; + tl = tl->next) + { + const struct TALER_Amount *amount_with_fee; + const struct TALER_Amount *fee; + const struct TALER_AmountNBO *fee_dki; + struct TALER_Amount tmp; + + switch (tl->type) + { + case TALER_EXCHANGEDB_TT_DEPOSIT: + /* check wire and h_wire are consistent */ + { + struct GNUNET_HashCode hw; + + if (GNUNET_OK != + TALER_JSON_merchant_wire_signature_hash ( + tl->details.deposit->receiver_wire_account, + &hw)) + { + report_row_inconsistency ("deposits", + tl->serial_id, + "wire value malformed"); + } + else if (0 != + GNUNET_memcmp (&hw, + &tl->details.deposit->h_wire)) + { + report_row_inconsistency ("deposits", + tl->serial_id, + "h(wire) does not match wire"); + } + } + amount_with_fee = &tl->details.deposit->amount_with_fee; + fee = &tl->details.deposit->deposit_fee; + fee_dki = &issue->fee_deposit; + if (GNUNET_OK != + TALER_amount_add (&expenditures, + &expenditures, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + /* Check if this deposit is within the remit of the aggregation + we are investigating, if so, include it in the totals. */ + if ( (0 == GNUNET_memcmp (merchant_pub, + &tl->details.deposit->merchant_pub)) && + (0 == GNUNET_memcmp (h_contract_terms, + &tl->details.deposit->h_contract_terms)) ) + { + struct TALER_Amount amount_without_fee; + + if (GNUNET_OK != + TALER_amount_subtract (&amount_without_fee, + amount_with_fee, + fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (merchant_gain, + merchant_gain, + &amount_without_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Detected applicable deposit of %s\n", + TALER_amount2s (&amount_without_fee)); + deposit_fee = fee; + } + /* Check that the fees given in the transaction list and in dki match */ + TALER_amount_ntoh (&tmp, + fee_dki); + if (0 != + TALER_amount_cmp (&tmp, + fee)) + { + /* Disagreement in fee structure within DB, should be impossible! */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + break; + case TALER_EXCHANGEDB_TT_MELT: + amount_with_fee = &tl->details.melt->amount_with_fee; + fee = &tl->details.melt->melt_fee; + fee_dki = &issue->fee_refresh; + if (GNUNET_OK != + TALER_amount_add (&expenditures, + &expenditures, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + /* Check that the fees given in the transaction list and in dki match */ + TALER_amount_ntoh (&tmp, + fee_dki); + if (0 != + TALER_amount_cmp (&tmp, + fee)) + { + /* Disagreement in fee structure within DB, should be impossible! */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + break; + case TALER_EXCHANGEDB_TT_REFUND: + amount_with_fee = &tl->details.refund->refund_amount; + fee = &tl->details.refund->refund_fee; + fee_dki = &issue->fee_refund; + if (GNUNET_OK != + TALER_amount_add (&refunds, + &refunds, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&expenditures, + &expenditures, + fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + /* Check if this refund is within the remit of the aggregation + we are investigating, if so, include it in the totals. */ + if ( (0 == GNUNET_memcmp (merchant_pub, + &tl->details.refund->merchant_pub)) && + (0 == GNUNET_memcmp (h_contract_terms, + &tl->details.refund->h_contract_terms)) ) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Detected applicable refund of %s\n", + TALER_amount2s (amount_with_fee)); + if (GNUNET_OK != + TALER_amount_add (&merchant_loss, + &merchant_loss, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + refund_deposit_fee = GNUNET_YES; + } + /* Check that the fees given in the transaction list and in dki match */ + TALER_amount_ntoh (&tmp, + fee_dki); + if (0 != + TALER_amount_cmp (&tmp, + fee)) + { + /* Disagreement in fee structure within DB, should be impossible! */ + GNUNET_break (0); + return GNUNET_SYSERR; + } + break; + case TALER_EXCHANGEDB_TT_OLD_COIN_RECOUP: + amount_with_fee = &tl->details.old_coin_recoup->value; + if (GNUNET_OK != + TALER_amount_add (&refunds, + &refunds, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + break; + case TALER_EXCHANGEDB_TT_RECOUP: + amount_with_fee = &tl->details.recoup->value; + if (GNUNET_OK != + TALER_amount_add (&expenditures, + &expenditures, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + break; + case TALER_EXCHANGEDB_TT_RECOUP_REFRESH: + amount_with_fee = &tl->details.recoup_refresh->value; + if (GNUNET_OK != + TALER_amount_add (&expenditures, + &expenditures, + amount_with_fee)) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + break; + } + } /* for 'tl' */ + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Deposits without fees are %s\n", + TALER_amount2s (merchant_gain)); + + /* Calculate total balance change, i.e. expenditures (recoup, deposit, refresh) + minus refunds (refunds, recoup-to-old) */ + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Subtracting refunds of %s from coin value loss\n", + TALER_amount2s (&refunds)); + if (GNUNET_SYSERR == + TALER_amount_subtract (&spent, + &expenditures, + &refunds)) + { + /* refunds above expenditures? Bad! */ + report_coin_arithmetic_inconsistency ("refund (balance)", + coin_pub, + &expenditures, + &refunds, + 1); + return GNUNET_SYSERR; + } + + /* Now check that 'spent' is less or equal than the total coin value */ + TALER_amount_ntoh (&value, + &issue->value); + if (1 == TALER_amount_cmp (&spent, + &value)) + { + /* spent > value */ + report_coin_arithmetic_inconsistency ("spend", + coin_pub, + &spent, + &value, + -1); + return GNUNET_SYSERR; + } + + /* Finally, update @a merchant_gain by subtracting what he "lost" + from refunds */ + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Merchant 'loss' due to refunds is %s\n", + TALER_amount2s (&merchant_loss)); + *deposit_gain = *merchant_gain; + if ( (GNUNET_YES == refund_deposit_fee) && + (NULL != deposit_fee) ) + { + /* We had a /deposit operation AND a /refund operation, + and should thus not charge the merchant the /deposit fee */ + GNUNET_assert (GNUNET_OK == + TALER_amount_add (merchant_gain, + merchant_gain, + deposit_fee)); + } + if (GNUNET_SYSERR == + TALER_amount_subtract (&final_gain, + merchant_gain, + &merchant_loss)) + { + /* refunds above deposits? Bad! */ + report_coin_arithmetic_inconsistency ("refund (merchant)", + coin_pub, + merchant_gain, + &merchant_loss, + 1); + return GNUNET_SYSERR; + } + *merchant_gain = final_gain; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Final merchant gain after refunds is %s\n", + TALER_amount2s (deposit_gain)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Coin %s contributes %s to contract %s\n", + TALER_B2S (coin_pub), + TALER_amount2s (merchant_gain), + GNUNET_h2s (h_contract_terms)); + return GNUNET_OK; +} + + +/** + * Function called with the results of the lookup of the + * transaction data associated with a wire transfer identifier. + * + * @param cls a `struct WireCheckContext` + * @param rowid which row in the table is the information from (for diagnostics) + * @param merchant_pub public key of the merchant (should be same for all callbacks with the same @e cls) + * @param h_wire hash of wire transfer details of the merchant (should be same for all callbacks with the same @e cls) + * @param account_details where did we transfer the funds? + * @param exec_time execution time of the wire transfer (should be same for all callbacks with the same @e cls) + * @param h_contract_terms which proposal was this payment about + * @param denom_pub denomination of @a coin_pub + * @param coin_pub which public key was this payment about + * @param coin_value amount contributed by this coin in total (with fee), + * but excluding refunds by this coin + * @param deposit_fee applicable deposit fee for this coin, actual + * fees charged may differ if coin was refunded + */ +static void +wire_transfer_information_cb ( + void *cls, + uint64_t rowid, + const struct TALER_MerchantPublicKeyP *merchant_pub, + const struct GNUNET_HashCode *h_wire, + const json_t *account_details, + struct GNUNET_TIME_Absolute exec_time, + const struct GNUNET_HashCode *h_contract_terms, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *coin_value, + const struct TALER_Amount *deposit_fee) +{ + struct WireCheckContext *wcc = cls; + const struct TALER_DenominationKeyValidityPS *issue; + struct TALER_Amount computed_value; + struct TALER_Amount coin_value_without_fee; + struct TALER_Amount total_deposit_without_refunds; + struct TALER_EXCHANGEDB_TransactionList *tl; + struct TALER_CoinPublicInfo coin; + enum GNUNET_DB_QueryStatus qs; + struct GNUNET_HashCode hw; + + if (GNUNET_OK != + TALER_JSON_merchant_wire_signature_hash (account_details, + &hw)) + { + report_row_inconsistency ("aggregation", + rowid, + "failed to compute hash of given wire data"); + } + else if (0 != + GNUNET_memcmp (&hw, + h_wire)) + { + report_row_inconsistency ("aggregation", + rowid, + "database contains wrong hash code for wire details"); + } + + /* Obtain coin's transaction history */ + qs = TALER_ARL_edb->get_coin_transactions (TALER_ARL_edb->cls, + TALER_ARL_esession, + coin_pub, + GNUNET_YES, + &tl); + if ( (qs < 0) || + (NULL == tl) ) + { + wcc->qs = qs; + report_row_inconsistency ("aggregation", + rowid, + "no transaction history for coin claimed in aggregation"); + return; + } + qs = TALER_ARL_edb->get_known_coin (TALER_ARL_edb->cls, + TALER_ARL_esession, + coin_pub, + &coin); + if (qs < 0) + { + GNUNET_break (0); /* this should be a foreign key violation at this point! */ + wcc->qs = qs; + report_row_inconsistency ("aggregation", + rowid, + "could not get coin details for coin claimed in aggregation"); + return; + } + + qs = TALER_ARL_get_denomination_info_by_hash (&coin.denom_pub_hash, + &issue); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_CRYPTO_rsa_signature_free (coin.denom_sig.rsa_signature); + TALER_ARL_edb->free_coin_transaction_list (TALER_ARL_edb->cls, + tl); + if (0 == qs) + report_row_inconsistency ("aggregation", + rowid, + "could not find denomination key for coin claimed in aggregation"); + else + wcc->qs = qs; + return; + } + if (GNUNET_OK != + TALER_test_coin_valid (&coin, + denom_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "wire", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount (coin_value), + "key_pub", GNUNET_JSON_from_data_auto ( + &issue->denom_hash))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + coin_value)); + GNUNET_CRYPTO_rsa_signature_free (coin.denom_sig.rsa_signature); + TALER_ARL_edb->free_coin_transaction_list (TALER_ARL_edb->cls, + tl); + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + report_row_inconsistency ("deposit", + rowid, + "coin denomination signature invalid"); + return; + } + GNUNET_CRYPTO_rsa_signature_free (coin.denom_sig.rsa_signature); + coin.denom_sig.rsa_signature = NULL; /* just to be sure */ + GNUNET_assert (NULL != issue); /* mostly to help static analysis */ + /* Check transaction history to see if it supports aggregate + valuation */ + if (GNUNET_OK != + check_transaction_history_for_deposit (coin_pub, + h_contract_terms, + merchant_pub, + issue, + tl, + &computed_value, + & + total_deposit_without_refunds)) + { + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + report_row_inconsistency ("coin history", + rowid, + "failed to verify coin history (for deposit)"); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Coin contributes %s to aggregate (deposits after fees and refunds)\n", + TALER_amount2s (&computed_value)); + if (GNUNET_SYSERR == + TALER_amount_subtract (&coin_value_without_fee, + coin_value, + deposit_fee)) + { + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + report_amount_arithmetic_inconsistency ( + "aggregation (fee structure)", + rowid, + coin_value, + deposit_fee, + -1); + return; + } + if (0 != + TALER_amount_cmp (&total_deposit_without_refunds, + &coin_value_without_fee)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Expected coin contribution of %s to aggregate\n", + TALER_amount2s (&coin_value_without_fee)); + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + report_amount_arithmetic_inconsistency ( + "aggregation (contribution)", + rowid, + &coin_value_without_fee, + & + total_deposit_without_refunds, + -1); + } + TALER_ARL_edb->free_coin_transaction_list (TALER_ARL_edb->cls, + tl); + + /* Check other details of wire transfer match */ + if (0 != GNUNET_memcmp (h_wire, + &wcc->h_wire)) + { + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + report_row_inconsistency ("aggregation", + rowid, + "target of outgoing wire transfer do not match hash of wire from deposit"); + } + if (exec_time.abs_value_us != wcc->date.abs_value_us) + { + /* This should be impossible from database constraints */ + GNUNET_break (0); + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + report_row_inconsistency ("aggregation", + rowid, + "date given in aggregate does not match wire transfer date"); + } + + /* Add coin's contribution to total aggregate value */ + { + struct TALER_Amount res; + + if (GNUNET_OK != + TALER_amount_add (&res, + &wcc->total_deposits, + &computed_value)) + { + GNUNET_break (0); + wcc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return; + } + wcc->total_deposits = res; + } +} + + +/** + * Lookup the wire fee that the exchange charges at @a timestamp. + * + * @param ac context for caching the result + * @param method method of the wire plugin + * @param timestamp time for which we need the fee + * @return NULL on error (fee unknown) + */ +static const struct TALER_Amount * +get_wire_fee (struct AggregationContext *ac, + const char *method, + struct GNUNET_TIME_Absolute timestamp) +{ + struct WireFeeInfo *wfi; + struct WireFeeInfo *pos; + struct TALER_MasterSignatureP master_sig; + + /* Check if fee is already loaded in cache */ + for (pos = ac->fee_head; NULL != pos; pos = pos->next) + { + if ( (pos->start_date.abs_value_us <= timestamp.abs_value_us) && + (pos->end_date.abs_value_us > timestamp.abs_value_us) ) + return &pos->wire_fee; + if (pos->start_date.abs_value_us > timestamp.abs_value_us) + break; + } + + /* Lookup fee in exchange database */ + wfi = GNUNET_new (struct WireFeeInfo); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != + TALER_ARL_edb->get_wire_fee (TALER_ARL_edb->cls, + TALER_ARL_esession, + method, + timestamp, + &wfi->start_date, + &wfi->end_date, + &wfi->wire_fee, + &wfi->closing_fee, + &master_sig)) + { + GNUNET_break (0); + GNUNET_free (wfi); + return NULL; + } + + /* Check signature. (This is not terribly meaningful as the exchange can + easily make this one up, but it means that we have proof that the master + key was used for inconsistent wire fees if a merchant complains.) */ + { + struct TALER_MasterWireFeePS wf; + + wf.purpose.purpose = htonl (TALER_SIGNATURE_MASTER_WIRE_FEES); + wf.purpose.size = htonl (sizeof (wf)); + GNUNET_CRYPTO_hash (method, + strlen (method) + 1, + &wf.h_wire_method); + wf.start_date = GNUNET_TIME_absolute_hton (wfi->start_date); + wf.end_date = GNUNET_TIME_absolute_hton (wfi->end_date); + TALER_amount_hton (&wf.wire_fee, + &wfi->wire_fee); + TALER_amount_hton (&wf.closing_fee, + &wfi->closing_fee); + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_MASTER_WIRE_FEES, + &wf.purpose, + &master_sig.eddsa_signature, + &TALER_ARL_master_pub.eddsa_pub)) + { + report_row_inconsistency ("wire-fee", + timestamp.abs_value_us, + "wire fee signature invalid at given time"); + } + } + + /* Established fee, keep in sorted list */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Wire fee is %s starting at %s\n", + TALER_amount2s (&wfi->wire_fee), + GNUNET_STRINGS_absolute_time_to_string (wfi->start_date)); + if ( (NULL == pos) || + (NULL == pos->prev) ) + GNUNET_CONTAINER_DLL_insert (ac->fee_head, + ac->fee_tail, + wfi); + else + GNUNET_CONTAINER_DLL_insert_after (ac->fee_head, + ac->fee_tail, + pos->prev, + wfi); + /* Check non-overlaping fee invariant */ + if ( (NULL != wfi->prev) && + (wfi->prev->end_date.abs_value_us > wfi->start_date.abs_value_us) ) + { + TALER_ARL_report (report_fee_time_inconsistencies, + json_pack ("{s:s, s:s, s:o}", + "type", method, + "diagnostic", + "start date before previous end date", + "time", TALER_ARL_json_from_time_abs ( + wfi->start_date))); + } + if ( (NULL != wfi->next) && + (wfi->next->start_date.abs_value_us >= wfi->end_date.abs_value_us) ) + { + TALER_ARL_report (report_fee_time_inconsistencies, + json_pack ("{s:s, s:s, s:o}", + "type", method, + "diagnostic", + "end date date after next start date", + "time", TALER_ARL_json_from_time_abs ( + wfi->end_date))); + } + return &wfi->wire_fee; +} + + +/** + * Check that a wire transfer made by the exchange is valid + * (has matching deposits). + * + * @param cls a `struct AggregationContext` + * @param rowid identifier of the respective row in the database + * @param date timestamp of the wire transfer (roughly) + * @param wtid wire transfer subject + * @param wire wire transfer details of the receiver + * @param amount amount that was wired + * @return #GNUNET_OK to continue, #GNUNET_SYSERR to stop iteration + */ +static int +check_wire_out_cb (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute date, + const struct TALER_WireTransferIdentifierRawP *wtid, + const json_t *wire, + const struct TALER_Amount *amount) +{ + struct AggregationContext *ac = cls; + struct WireCheckContext wcc; + struct TALER_Amount final_amount; + struct TALER_Amount exchange_gain; + enum GNUNET_DB_QueryStatus qs; + char *method; + + /* should be monotonically increasing */ + GNUNET_assert (rowid >= ppa.last_wire_out_serial_id); + ppa.last_wire_out_serial_id = rowid + 1; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Checking wire transfer %s over %s performed on %s\n", + TALER_B2S (wtid), + TALER_amount2s (amount), + GNUNET_STRINGS_absolute_time_to_string (date)); + if (NULL == (method = TALER_JSON_wire_to_method (wire))) + { + report_row_inconsistency ("wire_out", + rowid, + "specified wire address lacks method"); + return GNUNET_OK; + } + + wcc.ac = ac; + wcc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + wcc.date = date; + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount->currency, + &wcc.total_deposits)); + if (GNUNET_OK != + TALER_JSON_merchant_wire_signature_hash (wire, + &wcc.h_wire)) + { + GNUNET_break (0); + GNUNET_free (method); + return GNUNET_SYSERR; + } + qs = TALER_ARL_edb->lookup_wire_transfer (TALER_ARL_edb->cls, + TALER_ARL_esession, + wtid, + &wire_transfer_information_cb, + &wcc); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + ac->qs = qs; + GNUNET_free (method); + return GNUNET_SYSERR; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != wcc.qs) + { + /* Note: detailed information was already logged + in #wire_transfer_information_cb, so here we + only log for debugging */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Inconsitency for wire_out %llu (WTID %s) detected\n", + (unsigned long long) rowid, + TALER_B2S (wtid)); + } + + + /* Subtract aggregation fee from total (if possible) */ + { + const struct TALER_Amount *wire_fee; + + wire_fee = get_wire_fee (ac, + method, + date); + if (NULL == wire_fee) + { + report_row_inconsistency ("wire-fee", + date.abs_value_us, + "wire fee unavailable for given time"); + /* If fee is unknown, we just assume the fee is zero */ + final_amount = wcc.total_deposits; + } + else if (GNUNET_SYSERR == + TALER_amount_subtract (&final_amount, + &wcc.total_deposits, + wire_fee)) + { + report_amount_arithmetic_inconsistency ( + "wire out (fee structure)", + rowid, + &wcc.total_deposits, + wire_fee, + -1); + /* If fee arithmetic fails, we just assume the fee is zero */ + final_amount = wcc.total_deposits; + } + } + GNUNET_free (method); + + /* Round down to amount supported by wire method */ + GNUNET_break (GNUNET_SYSERR != + TALER_amount_round_down (&final_amount, + &TALER_ARL_currency_round_unit)); + + /* Calculate the exchange's gain as the fees plus rounding differences! */ + if (GNUNET_SYSERR == + TALER_amount_subtract (&exchange_gain, + &wcc.total_deposits, + &final_amount)) + { + GNUNET_break (0); + ac->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + + /* Sum up aggregation fees (we simply include the rounding gains) */ + if (GNUNET_OK != + TALER_amount_add (&total_aggregation_fee_income, + &total_aggregation_fee_income, + &exchange_gain)) + { + GNUNET_break (0); + ac->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + + /* Check that calculated amount matches actual amount */ + if (0 != TALER_amount_cmp (amount, + &final_amount)) + { + struct TALER_Amount delta; + + if (0 < TALER_amount_cmp (amount, + &final_amount)) + { + /* amount > final_amount */ + GNUNET_assert (GNUNET_OK == + TALER_amount_subtract (&delta, + amount, + &final_amount)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&total_wire_out_delta_plus, + &total_wire_out_delta_plus, + &delta)); + } + else + { + /* amount < final_amount */ + GNUNET_assert (GNUNET_OK == + TALER_amount_subtract (&delta, + &final_amount, + amount)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&total_wire_out_delta_minus, + &total_wire_out_delta_minus, + &delta)); + } + + TALER_ARL_report (report_wire_out_inconsistencies, + json_pack ("{s:O, s:I, s:o, s:o}", + "destination_account", wire, + "rowid", (json_int_t) rowid, + "expected", + TALER_JSON_from_amount (&final_amount), + "claimed", + TALER_JSON_from_amount (amount))); + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Wire transfer %s is OK\n", + TALER_B2S (wtid)); + return GNUNET_OK; +} + + +/** + * Analyze the exchange aggregator's payment processing. + * + * @param cls closure + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +analyze_aggregations (void *cls) +{ + struct AggregationContext ac; + struct WireFeeInfo *wfi; + enum GNUNET_DB_QueryStatus qsx; + enum GNUNET_DB_QueryStatus qs; + enum GNUNET_DB_QueryStatus qsp; + + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Analyzing aggregations\n"); + qsp = TALER_ARL_adb->get_auditor_progress_aggregation (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppa); + if (0 > qsp) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); + return qsp; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) + { + GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, + _ ( + "First analysis using this auditor, starting audit from scratch\n")); + } + else + { + ppa_start = ppa; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Resuming aggregation audit at %llu\n"), + (unsigned long long) ppa.last_wire_out_serial_id); + } + + memset (&ac, + 0, + sizeof (ac)); + qsx = TALER_ARL_adb->get_wire_fee_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_aggregation_fee_income); + if (0 > qsx) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); + return qsx; + } + ac.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + qs = TALER_ARL_edb->select_wire_out_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppa. + last_wire_out_serial_id, + &check_wire_out_cb, + &ac); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + ac.qs = qs; + } + while (NULL != (wfi = ac.fee_head)) + { + GNUNET_CONTAINER_DLL_remove (ac.fee_head, + ac.fee_tail, + wfi); + GNUNET_free (wfi); + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + /* there were no wire out entries to be looked at, we are done */ + return qs; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ac.qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == ac.qs); + return ac.qs; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsx) + ac.qs = TALER_ARL_adb->insert_wire_fee_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + & + total_aggregation_fee_income); + else + ac.qs = TALER_ARL_adb->update_wire_fee_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + & + total_aggregation_fee_income); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != ac.qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == ac.qs); + return ac.qs; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) + qs = TALER_ARL_adb->update_auditor_progress_aggregation (TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + &ppa); + else + qs = TALER_ARL_adb->insert_auditor_progress_aggregation (TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + &ppa); + if (0 >= qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to update auditor DB, not recording progress\n"); + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Concluded aggregation audit step at %llu\n"), + (unsigned long long) ppa.last_wire_out_serial_id); + + return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; +} + + +/** + * Main function that will be run. + * + * @param cls closure + * @param args remaining command-line arguments + * @param TALER_ARL_cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *TALER_ARL_cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + json_t *report; + + (void) cls; + (void) args; + (void) TALER_ARL_cfgfile; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Launching auditor\n"); + if (GNUNET_OK != + TALER_ARL_init (c)) + { + global_ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting audit\n"); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_aggregation_fee_income)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_wire_out_delta_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_wire_out_delta_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_arithmetic_delta_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_arithmetic_delta_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_coin_delta_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_coin_delta_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_sig_loss)); + GNUNET_assert (NULL != + (report_row_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_wire_out_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_coin_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_amount_arithmetic_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_bad_sig_losses = json_array ())); + GNUNET_assert (NULL != + (report_fee_time_inconsistencies = json_array ())); + TALER_ARL_setup_sessions_and_run (&analyze_aggregations, + NULL); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Audit complete\n"); + report = json_pack ("{s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:I, s:I," + " s:o, s:o }", + /* blocks #1 */ + "wire_out_inconsistencies", + report_wire_out_inconsistencies, + "total_wire_out_delta_plus", + TALER_JSON_from_amount ( + &total_wire_out_delta_plus), + "total_wire_out_delta_minus", + TALER_JSON_from_amount ( + &total_wire_out_delta_minus), + /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ + "bad_sig_losses", + report_bad_sig_losses, + /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ + "total_bad_sig_loss", + TALER_JSON_from_amount (&total_bad_sig_loss), + /* block #2 */ + /* Tested in test-auditor.sh #14/#15 */ + "row_inconsistencies", + report_row_inconsistencies, + "coin_inconsistencies", + report_coin_inconsistencies, + "total_coin_delta_plus", + TALER_JSON_from_amount (&total_coin_delta_plus), + "total_coin_delta_minus", + TALER_JSON_from_amount ( + &total_coin_delta_minus), + "amount_arithmetic_inconsistencies", + report_amount_arithmetic_inconsistencies, + /* block #3 */ + "total_arithmetic_delta_plus", + TALER_JSON_from_amount ( + &total_arithmetic_delta_plus), + "total_arithmetic_delta_minus", + TALER_JSON_from_amount ( + &total_arithmetic_delta_minus), + "total_aggregation_fee_income", + TALER_JSON_from_amount ( + &total_aggregation_fee_income), + "start_ppa_wire_out_serial_id", + (json_int_t) ppa_start.last_wire_out_serial_id, + "end_ppa_wire_out_serial_id", + (json_int_t) ppa.last_wire_out_serial_id, + /* block #4 */ + "auditor_start_time", + TALER_ARL_json_from_time_abs ( + start_time), + "auditor_end_time", + TALER_ARL_json_from_time_abs ( + GNUNET_TIME_absolute_get ()) + ); + GNUNET_break (NULL != report); + TALER_ARL_done (report); +} + + +/** + * The main function of the database initialization tool. + * Used to initialize the Taler Exchange's database. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + const struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_base32_auto ('m', + "exchange-key", + "KEY", + "public key of the exchange (Crockford base32 encoded)", + &TALER_ARL_master_pub), + GNUNET_GETOPT_option_flag ('r', + "TALER_ARL_restart", + "TALER_ARL_restart audit from the beginning (required on first run)", + &TALER_ARL_restart), + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_OPTION_END + }; + + /* force linker to link against libtalerutil; if we do + not do this, the linker may "optimize" libtalerutil + away and skip #TALER_OS_init(), which we do need */ + (void) TALER_project_data_default (); + GNUNET_assert (GNUNET_OK == + GNUNET_log_setup ("taler-auditor-aggregation", + "MESSAGE", + NULL)); + if (GNUNET_OK != + GNUNET_PROGRAM_run (argc, + argv, + "taler-auditor-aggregation", + "Audit Taler exchange aggregation activity", + options, + &run, + NULL)) + return 1; + return global_ret; +} + + +/* end of taler-auditor-aggregation.c */ diff --git a/src/auditor/taler-helper-auditor-coins.c b/src/auditor/taler-helper-auditor-coins.c new file mode 100644 index 000000000..c9250961b --- /dev/null +++ b/src/auditor/taler-helper-auditor-coins.c @@ -0,0 +1,2374 @@ +/* + This file is part of TALER + Copyright (C) 2016-2020 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. + + You should have received a copy of the GNU Affero Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file auditor/taler-helper-auditor0coins.c + * @brief audits coins in an exchange database. + * @author Christian Grothoff + * + * UNDECIDED: + * - do we care about checking the 'done' flag in deposit_cb? + */ +#include "platform.h" +#include +#include "taler_auditordb_plugin.h" +#include "taler_exchangedb_lib.h" +#include "taler_json_lib.h" +#include "taler_bank_service.h" +#include "taler_signatures.h" +#include "report-lib.h" + +/** + * How many coin histories do we keep in RAM at any given point in + * time? Used bound memory consumption of the auditor. Larger values + * reduce database accesses. + * + * Set to a VERY low value here for testing. Practical values may be + * in the millions. + */ +#define MAX_COIN_SUMMARIES 4 + +/** + * Use a 1 day grace period to deal with clocks not being perfectly synchronized. + */ +#define DEPOSIT_GRACE_PERIOD GNUNET_TIME_UNIT_DAYS + +/** + * Return value from main(). + */ +static int global_ret; + +/** + * Checkpointing our progress for coins. + */ +static struct TALER_AUDITORDB_ProgressPointCoin ppc; + +/** + * Checkpointing our progress for coins. + */ +static struct TALER_AUDITORDB_ProgressPointCoin ppc_start; + +/** + * Array of TALER_ARL_reports about denomination keys with an + * emergency (more value deposited than withdrawn) + */ +static json_t *report_emergencies; + +/** + * Array of TALER_ARL_reports about denomination keys with an + * emergency (more coins deposited than withdrawn) + */ +static json_t *report_emergencies_by_count; + +/** + * Array of TALER_ARL_reports about row inconsitencies. + */ +static json_t *report_row_inconsistencies; + +/** + * Report about amount calculation differences (causing profit + * or loss at the exchange). + */ +static json_t *report_amount_arithmetic_inconsistencies; + +/** + * Profits the exchange made by bad amount calculations. + */ +static struct TALER_Amount total_arithmetic_delta_plus; + +/** + * Losses the exchange made by bad amount calculations. + */ +static struct TALER_Amount total_arithmetic_delta_minus; + +/** + * Total amount reported in all calls to #report_emergency_by_count(). + */ +static struct TALER_Amount reported_emergency_risk_by_count; + +/** + * Total amount reported in all calls to #report_emergency_by_amount(). + */ +static struct TALER_Amount reported_emergency_risk_by_amount; + +/** + * Total amount in losses reported in all calls to #report_emergency_by_amount(). + */ +static struct TALER_Amount reported_emergency_loss; + +/** + * Total amount in losses reported in all calls to #report_emergency_by_count(). + */ +static struct TALER_Amount reported_emergency_loss_by_count; + +/** + * Expected balance in the escrow account. + */ +static struct TALER_Amount total_escrow_balance; + +/** + * Active risk exposure. + */ +static struct TALER_Amount total_risk; + +/** + * Actualized risk (= loss) from recoups. + */ +static struct TALER_Amount total_recoup_loss; + +/** + * Recoups we made on denominations that were not revoked (!?). + */ +static struct TALER_Amount total_irregular_recoups; + +/** + * Total deposit fees earned. + */ +static struct TALER_Amount total_deposit_fee_income; + +/** + * Total melt fees earned. + */ +static struct TALER_Amount total_melt_fee_income; + +/** + * Total refund fees earned. + */ +static struct TALER_Amount total_refund_fee_income; + +/** + * Array of TALER_ARL_reports about coin operations with bad signatures. + */ +static json_t *report_bad_sig_losses; + +/** + * Total amount lost by operations for which signatures were invalid. + */ +static struct TALER_Amount total_bad_sig_loss; + +/** + * Array of refresh transactions where the /refresh/reveal has not yet + * happened (and may of course never happen). + */ +static json_t *report_refreshs_hanging; + +/** + * Total amount lost by operations for which signatures were invalid. + */ +static struct TALER_Amount total_refresh_hanging; + + +/* ***************************** Report logic **************************** */ + +/** + * Called in case we detect an emergency situation where the exchange + * is paying out a larger amount on a denomination than we issued in + * that denomination. This means that the exchange's private keys + * might have gotten compromised, and that we need to trigger an + * emergency request to all wallets to deposit pending coins for the + * denomination (and as an exchange suffer a huge financial loss). + * + * @param issue denomination key where the loss was detected + * @param risk maximum risk that might have just become real (coins created by this @a issue) + * @param loss actual losses already (actualized before denomination was revoked) + */ +static void +report_emergency_by_amount (const struct + TALER_DenominationKeyValidityPS *issue, + const struct TALER_Amount *risk, + const struct TALER_Amount *loss) +{ + TALER_ARL_report (report_emergencies, + json_pack ("{s:o, s:o, s:o, s:o, s:o, s:o}", + "denompub_hash", + GNUNET_JSON_from_data_auto (&issue->denom_hash), + "denom_risk", + TALER_JSON_from_amount (risk), + "denom_loss", + TALER_JSON_from_amount (loss), + "start", + TALER_ARL_TALER_ARL_json_from_time_abs_nbo ( + issue->start), + "deposit_end", + TALER_ARL_TALER_ARL_json_from_time_abs_nbo ( + issue->expire_deposit), + "value", + TALER_JSON_from_amount_nbo (&issue->value))); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&reported_emergency_risk_by_amount, + &reported_emergency_risk_by_amount, + risk)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&reported_emergency_loss, + &reported_emergency_loss, + loss)); +} + + +/** + * Called in case we detect an emergency situation where the exchange + * is paying out a larger NUMBER of coins of a denomination than we + * issued in that denomination. This means that the exchange's + * private keys might have gotten compromised, and that we need to + * trigger an emergency request to all wallets to deposit pending + * coins for the denomination (and as an exchange suffer a huge + * financial loss). + * + * @param issue denomination key where the loss was detected + * @param num_issued number of coins that were issued + * @param num_known number of coins that have been deposited + * @param risk amount that is at risk + */ +static void +report_emergency_by_count (const struct + TALER_DenominationKeyValidityPS *issue, + uint64_t num_issued, + uint64_t num_known, + const struct TALER_Amount *risk) +{ + struct TALER_Amount denom_value; + + TALER_ARL_report (report_emergencies_by_count, + json_pack ("{s:o, s:I, s:I, s:o, s:o, s:o, s:o}", + "denompub_hash", + GNUNET_JSON_from_data_auto (&issue->denom_hash), + "num_issued", + (json_int_t) num_issued, + "num_known", + (json_int_t) num_known, + "denom_risk", + TALER_JSON_from_amount (risk), + "start", + TALER_ARL_TALER_ARL_json_from_time_abs_nbo ( + issue->start), + "deposit_end", + TALER_ARL_TALER_ARL_json_from_time_abs_nbo ( + issue->expire_deposit), + "value", + TALER_JSON_from_amount_nbo (&issue->value))); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&reported_emergency_risk_by_count, + &reported_emergency_risk_by_count, + risk)); + TALER_amount_ntoh (&denom_value, + &issue->value); + for (uint64_t i = num_issued; i auditor */ + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + exchange, + auditor)); + } + else + { + /* auditor < exchange */ + profitable = -profitable; + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + auditor, + exchange)); + } + TALER_ARL_report (report_amount_arithmetic_inconsistencies, + json_pack ("{s:s, s:I, s:o, s:o, s:I}", + "operation", operation, + "rowid", (json_int_t) rowid, + "exchange", TALER_JSON_from_amount (exchange), + "auditor", TALER_JSON_from_amount (auditor), + "profitable", (json_int_t) profitable)); + if (0 != profitable) + { + target = (1 == profitable) + ? &total_arithmetic_delta_plus + : &total_arithmetic_delta_minus; + GNUNET_break (GNUNET_OK == + TALER_amount_add (target, + target, + &delta)); + } +} + + +/** + * Report a (serious) inconsistency in the exchange's database. + * + * @param table affected table + * @param rowid affected row, UINT64_MAX if row is missing + * @param diagnostic message explaining the problem + */ +static void +report_row_inconsistency (const char *table, + uint64_t rowid, + const char *diagnostic) +{ + TALER_ARL_report (report_row_inconsistencies, + json_pack ("{s:s, s:I, s:s}", + "table", table, + "row", (json_int_t) rowid, + "diagnostic", diagnostic)); +} + + +/* ************************* Analyze coins ******************** */ +/* This logic checks that the exchange did the right thing for each + coin, checking deposits, refunds, refresh* and known_coins + tables */ + + +/** + * Summary data we keep per denomination. + */ +struct DenominationSummary +{ + /** + * Total value of outstanding (not deposited) coins issued with this + * denomination key. + */ + struct TALER_Amount denom_balance; + + /** + * Total losses made (once coins deposited exceed + * coins withdrawn and thus the @e denom_balance is + * effectively negative). + */ + struct TALER_Amount denom_loss; + + /** + * Total value of coins issued with this denomination key. + */ + struct TALER_Amount denom_risk; + + /** + * Total value of coins subjected to recoup with this denomination key. + */ + struct TALER_Amount denom_recoup; + + /** + * How many coins (not their amount!) of this denomination + * did the exchange issue overall? + */ + uint64_t num_issued; + + /** + * Denomination key information for this denomination. + */ + const struct TALER_DenominationKeyValidityPS *issue; + + /** + * #GNUNET_YES if this record already existed in the DB. + * Used to decide between insert/update in + * #sync_denomination(). + */ + int in_db; + + /** + * Should we TALER_ARL_report an emergency for this denomination? + */ + int report_emergency; + + /** + * #GNUNET_YES if this denomination was revoked. + */ + int was_revoked; +}; + + +/** + * Closure for callbacks during #analyze_coins(). + */ +struct CoinContext +{ + + /** + * Map for tracking information about denominations. + */ + struct GNUNET_CONTAINER_MultiHashMap *denom_summaries; + + /** + * Current write/replace offset in the circular @e summaries buffer. + */ + unsigned int summaries_off; + + /** + * Transaction status code. + */ + enum GNUNET_DB_QueryStatus qs; + +}; + + +/** + * Initialize information about denomination from the database. + * + * @param denom_hash hash of the public key of the denomination + * @param[out] ds summary to initialize + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +init_denomination (const struct GNUNET_HashCode *denom_hash, + struct DenominationSummary *ds) +{ + enum GNUNET_DB_QueryStatus qs; + struct TALER_MasterSignatureP msig; + uint64_t rowid; + + qs = TALER_ARL_adb->get_denomination_balance (TALER_ARL_adb->cls, + TALER_ARL_asession, + denom_hash, + &ds->denom_balance, + &ds->denom_loss, + &ds->denom_risk, + &ds->denom_recoup, + &ds->num_issued); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) + { + ds->in_db = GNUNET_YES; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting balance for denomination `%s' is %s\n", + GNUNET_h2s (denom_hash), + TALER_amount2s (&ds->denom_balance)); + return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + } + qs = TALER_ARL_edb->get_denomination_revocation (TALER_ARL_edb->cls, + TALER_ARL_esession, + denom_hash, + &msig, + &rowid); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 < qs) + { + /* check revocation signature */ + struct TALER_MasterDenominationKeyRevocationPS rm; + + rm.purpose.purpose = htonl ( + TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED); + rm.purpose.size = htonl (sizeof (rm)); + rm.h_denom_pub = *denom_hash; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify ( + TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED, + &rm.purpose, + &msig.eddsa_signature, + &TALER_ARL_master_pub.eddsa_pub)) + { + report_row_inconsistency ("denomination revocation table", + rowid, + "revocation signature invalid"); + } + else + { + ds->was_revoked = GNUNET_YES; + } + } + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &ds->denom_balance)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &ds->denom_loss)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &ds->denom_risk)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &ds->denom_recoup)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting balance for denomination `%s' is %s\n", + GNUNET_h2s (denom_hash), + TALER_amount2s (&ds->denom_balance)); + return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; +} + + +/** + * Obtain the denomination summary for the given @a dh + * + * @param cc our execution context + * @param issue denomination key information for @a dh + * @param dh the denomination hash to use for the lookup + * @return NULL on error + */ +static struct DenominationSummary * +get_denomination_summary (struct CoinContext *cc, + const struct TALER_DenominationKeyValidityPS *issue, + const struct GNUNET_HashCode *dh) +{ + struct DenominationSummary *ds; + + ds = GNUNET_CONTAINER_multihashmap_get (cc->denom_summaries, + dh); + if (NULL != ds) + return ds; + ds = GNUNET_new (struct DenominationSummary); + ds->issue = issue; + if (0 > (cc->qs = init_denomination (dh, + ds))) + { + GNUNET_break (0); + GNUNET_free (ds); + return NULL; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (cc->denom_summaries, + dh, + ds, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + return ds; +} + + +/** + * Write information about the current knowledge about a denomination key + * back to the database and update our global TALER_ARL_reporting data about the + * denomination. Also remove and free the memory of @a value. + * + * @param cls the `struct CoinContext` + * @param denom_hash the hash of the denomination key + * @param value a `struct DenominationSummary` + * @return #GNUNET_OK (continue to iterate) + */ +static int +sync_denomination (void *cls, + const struct GNUNET_HashCode *denom_hash, + void *value) +{ + struct CoinContext *cc = cls; + struct DenominationSummary *ds = value; + const struct TALER_DenominationKeyValidityPS *issue = ds->issue; + struct GNUNET_TIME_Absolute now; + struct GNUNET_TIME_Absolute expire_deposit; + struct GNUNET_TIME_Absolute expire_deposit_grace; + enum GNUNET_DB_QueryStatus qs; + + now = GNUNET_TIME_absolute_get (); + expire_deposit = GNUNET_TIME_absolute_ntoh (issue->expire_deposit); + /* add day grace period to deal with clocks not being perfectly synchronized */ + expire_deposit_grace = GNUNET_TIME_absolute_add (expire_deposit, + DEPOSIT_GRACE_PERIOD); + if (now.abs_value_us > expire_deposit_grace.abs_value_us) + { + /* Denominationkey has expired, book remaining balance of + outstanding coins as revenue; and reduce cc->risk exposure. */ + if (ds->in_db) + qs = TALER_ARL_adb->del_denomination_balance (TALER_ARL_adb->cls, + TALER_ARL_asession, + denom_hash); + else + qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + if ( (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) && + ( (0 != ds->denom_risk.value) || + (0 != ds->denom_risk.fraction) ) ) + { + /* The denomination expired and carried a balance; we can now + book the remaining balance as profit, and reduce our risk + exposure by the accumulated risk of the denomination. */ + if (GNUNET_SYSERR == + TALER_amount_subtract (&total_risk, + &total_risk, + &ds->denom_risk)) + { + /* Holy smokes, our risk assessment was inconsistent! + This is really, really bad. */ + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + } + } + if ( (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) && + ( (0 != ds->denom_balance.value) || + (0 != ds->denom_balance.fraction) ) ) + { + /* book denom_balance coin expiration profits! */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Denomination `%s' expired, booking %s in expiration profits\n", + GNUNET_h2s (denom_hash), + TALER_amount2s (&ds->denom_balance)); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != + (qs = TALER_ARL_adb->insert_historic_denom_revenue ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + denom_hash, + expire_deposit, + &ds->denom_balance, + &ds->denom_recoup))) + { + /* Failed to store profits? Bad database */ + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + } + } + } + else + { + long long cnt; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Final balance for denomination `%s' is %s (%llu)\n", + GNUNET_h2s (denom_hash), + TALER_amount2s (&ds->denom_balance), + (unsigned long long) ds->num_issued); + cnt = TALER_ARL_edb->count_known_coins (TALER_ARL_edb->cls, + TALER_ARL_esession, + denom_hash); + if (0 > cnt) + { + /* Failed to obtain count? Bad database */ + qs = (enum GNUNET_DB_QueryStatus) cnt; + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + } + else + { + if (ds->num_issued < (uint64_t) cnt) + { + report_emergency_by_count (issue, + ds->num_issued, + cnt, + &ds->denom_risk); + } + if (GNUNET_YES == ds->report_emergency) + { + report_emergency_by_amount (issue, + &ds->denom_risk, + &ds->denom_loss); + + } + if (ds->in_db) + qs = TALER_ARL_adb->update_denomination_balance (TALER_ARL_adb->cls, + TALER_ARL_asession, + denom_hash, + &ds->denom_balance, + &ds->denom_loss, + &ds->denom_risk, + &ds->denom_recoup, + ds->num_issued); + else + qs = TALER_ARL_adb->insert_denomination_balance (TALER_ARL_adb->cls, + TALER_ARL_asession, + denom_hash, + &ds->denom_balance, + &ds->denom_loss, + &ds->denom_risk, + &ds->denom_recoup, + ds->num_issued); + } + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + } + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (cc->denom_summaries, + denom_hash, + ds)); + GNUNET_free (ds); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != cc->qs) + return GNUNET_SYSERR; + return GNUNET_OK; +} + + +/** + * Function called with details about all withdraw operations. + * Updates the denomination balance and the overall balance as + * we now have additional coins that have been issued. + * + * Note that the signature was already checked in + * #handle_reserve_out(), so we do not check it again here. + * + * @param cls our `struct CoinContext` + * @param rowid unique serial ID for the refresh session in our DB + * @param h_blind_ev blinded hash of the coin's public key + * @param denom_pub public denomination key of the deposited coin + * @param reserve_pub public key of the reserve + * @param reserve_sig signature over the withdraw operation (verified elsewhere) + * @param execution_date when did the wallet withdraw the coin + * @param amount_with_fee amount that was withdrawn + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +withdraw_cb (void *cls, + uint64_t rowid, + const struct GNUNET_HashCode *h_blind_ev, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_ReservePublicKeyP *reserve_pub, + const struct TALER_ReserveSignatureP *reserve_sig, + struct GNUNET_TIME_Absolute execution_date, + const struct TALER_Amount *amount_with_fee) +{ + struct CoinContext *cc = cls; + struct DenominationSummary *ds; + struct GNUNET_HashCode dh; + const struct TALER_DenominationKeyValidityPS *issue; + struct TALER_Amount value; + enum GNUNET_DB_QueryStatus qs; + + (void) h_blind_ev; + (void) reserve_pub; + (void) reserve_sig; + (void) execution_date; + (void) amount_with_fee; + GNUNET_assert (rowid >= ppc.last_withdraw_serial_id); /* should be monotonically increasing */ + ppc.last_withdraw_serial_id = rowid + 1; + + qs = TALER_ARL_get_denomination_info (denom_pub, + &issue, + &dh); + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("withdraw", + rowid, + "denomination key not found"); + return GNUNET_OK; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + /* This really ought to be a transient DB error. */ + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + return GNUNET_SYSERR; + } + ds = get_denomination_summary (cc, + issue, + &dh); + if (NULL == ds) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + TALER_amount_ntoh (&value, + &issue->value); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Issued coin in denomination `%s' of total value %s\n", + GNUNET_h2s (&dh), + TALER_amount2s (&value)); + ds->num_issued++; + if (GNUNET_OK != + TALER_amount_add (&ds->denom_balance, + &ds->denom_balance, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "New balance of denomination `%s' is %s\n", + GNUNET_h2s (&dh), + TALER_amount2s (&ds->denom_balance)); + if (GNUNET_OK != + TALER_amount_add (&total_escrow_balance, + &total_escrow_balance, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&total_risk, + &total_risk, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&ds->denom_risk, + &ds->denom_risk, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + return GNUNET_OK; +} + + +/** + * Closure for #reveal_data_cb(). + */ +struct RevealContext +{ + + /** + * Denomination public keys of the new coins. + */ + struct TALER_DenominationPublicKey *new_dps; + + /** + * Size of the @a new_dp and @a new_dps arrays. + */ + unsigned int num_freshcoins; +}; + + +/** + * Function called with information about a refresh order. + * + * @param cls closure + * @param num_freshcoins size of the @a rrcs array + * @param rrcs array of @a num_freshcoins information about coins to be created + * @param num_tprivs number of entries in @a tprivs, should be #TALER_CNC_KAPPA - 1 + * @param tprivs array of @e num_tprivs transfer private keys + * @param tp transfer public key information + */ +static void +reveal_data_cb (void *cls, + uint32_t num_freshcoins, + const struct TALER_EXCHANGEDB_RefreshRevealedCoin *rrcs, + unsigned int num_tprivs, + const struct TALER_TransferPrivateKeyP *tprivs, + const struct TALER_TransferPublicKeyP *tp) +{ + struct RevealContext *rctx = cls; + + (void) num_tprivs; + (void) tprivs; + (void) tp; + rctx->num_freshcoins = num_freshcoins; + rctx->new_dps = GNUNET_new_array (num_freshcoins, + struct TALER_DenominationPublicKey); + for (unsigned int i = 0; inew_dps[i].rsa_public_key + = GNUNET_CRYPTO_rsa_public_key_dup (rrcs[i].denom_pub.rsa_public_key); +} + + +/** + * Check that the @a coin_pub is a known coin with a proper + * signature for denominatinon @a denom_pub. If not, TALER_ARL_report + * a loss of @a loss_potential. + * + * @param coin_pub public key of a coin + * @param denom_pub expected denomination of the coin + * @param loss_potential how big could the loss be if the coin is + * not properly signed + * @return database transaction status, on success + * #GNUNET_DB_STATUS_SUCCESS_ONE_RESULT + */ +static enum GNUNET_DB_QueryStatus +check_known_coin (const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_Amount *loss_potential) +{ + struct TALER_CoinPublicInfo ci; + enum GNUNET_DB_QueryStatus qs; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Checking denomination signature on %s\n", + TALER_B2S (coin_pub)); + qs = TALER_ARL_edb->get_known_coin (TALER_ARL_edb->cls, + TALER_ARL_esession, + coin_pub, + &ci); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (GNUNET_YES != + TALER_test_coin_valid (&ci, + denom_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "known-coin", + "row", (json_int_t) -1, + "loss", TALER_JSON_from_amount ( + loss_potential), + "key_pub", GNUNET_JSON_from_data_auto ( + coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + loss_potential)); + + } + GNUNET_CRYPTO_rsa_signature_free (ci.denom_sig.rsa_signature); + return qs; +} + + +/** + * Function called with details about coins that were melted, with the + * goal of auditing the refresh's execution. Verifies the signature + * and updates our information about coins outstanding (the old coin's + * denomination has less, the fresh coins increased outstanding + * balances). + * + * @param cls closure + * @param rowid unique serial ID for the refresh session in our DB + * @param denom_pub denomination public key of @a coin_pub + * @param coin_pub public key of the coin + * @param coin_sig signature from the coin + * @param amount_with_fee amount that was deposited including fee + * @param noreveal_index which index was picked by the exchange in cut-and-choose + * @param rc what is the refresh commitment + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +refresh_session_cb (void *cls, + uint64_t rowid, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_CoinSpendSignatureP *coin_sig, + const struct TALER_Amount *amount_with_fee, + uint32_t noreveal_index, + const struct TALER_RefreshCommitmentP *rc) +{ + struct CoinContext *cc = cls; + struct TALER_RefreshMeltCoinAffirmationPS rmc; + const struct TALER_DenominationKeyValidityPS *issue; + struct DenominationSummary *dso; + struct TALER_Amount amount_without_fee; + struct TALER_Amount tmp; + enum GNUNET_DB_QueryStatus qs; + + (void) noreveal_index; + GNUNET_assert (rowid >= ppc.last_melt_serial_id); /* should be monotonically increasing */ + ppc.last_melt_serial_id = rowid + 1; + + qs = TALER_ARL_get_denomination_info (denom_pub, + &issue, + NULL); + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("melt", + rowid, + "denomination key not found"); + return GNUNET_OK; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + return GNUNET_SYSERR; + } + qs = check_known_coin (coin_pub, + denom_pub, + amount_with_fee); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + return GNUNET_SYSERR; + } + + /* verify melt signature */ + rmc.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_MELT); + rmc.purpose.size = htonl (sizeof (rmc)); + rmc.rc = *rc; + TALER_amount_hton (&rmc.amount_with_fee, + amount_with_fee); + rmc.melt_fee = issue->fee_refresh; + rmc.coin_pub = *coin_pub; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_MELT, + &rmc.purpose, + &coin_sig->eddsa_signature, + &coin_pub->eddsa_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "melt", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount ( + amount_with_fee), + "key_pub", GNUNET_JSON_from_data_auto ( + coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount_with_fee)); + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Melting coin %s in denomination `%s' of value %s\n", + TALER_B2S (coin_pub), + GNUNET_h2s (&issue->denom_hash), + TALER_amount2s (amount_with_fee)); + + { + struct RevealContext reveal_ctx; + struct TALER_Amount refresh_cost; + int err; + + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount_with_fee->currency, + &refresh_cost)); + memset (&reveal_ctx, + 0, + sizeof (reveal_ctx)); + qs = TALER_ARL_edb->get_refresh_reveal (TALER_ARL_edb->cls, + TALER_ARL_esession, + rc, + &reveal_data_cb, + &reveal_ctx); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if ( (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) || + (0 == reveal_ctx.num_freshcoins) ) + { + /* This can happen if /refresh/reveal was not yet called or only + with invalid data, even if the exchange is correctly + operating. We still TALER_ARL_report it. */ + TALER_ARL_report (report_refreshs_hanging, + json_pack ("{s:I, s:o, s:o}", + "row", (json_int_t) rowid, + "amount", TALER_JSON_from_amount ( + amount_with_fee), + "coin_pub", GNUNET_JSON_from_data_auto ( + coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_refresh_hanging, + &total_refresh_hanging, + amount_with_fee)); + return GNUNET_OK; + } + + { + const struct TALER_DenominationKeyValidityPS *new_issues[reveal_ctx. + num_freshcoins]; + + /* Update outstanding amounts for all new coin's denominations, and check + that the resulting amounts are consistent with the value being refreshed. */ + err = GNUNET_OK; + for (unsigned int i = 0; iqs = qs; + err = GNUNET_SYSERR; /* terminate, return GNUNET_SYSERR */ + } + GNUNET_CRYPTO_rsa_public_key_free ( + reveal_ctx.new_dps[i].rsa_public_key); + reveal_ctx.new_dps[i].rsa_public_key = NULL; + } + GNUNET_free (reveal_ctx.new_dps); + reveal_ctx.new_dps = NULL; + + if (GNUNET_OK != err) + return (GNUNET_SYSERR == err) ? GNUNET_SYSERR : GNUNET_OK; + + /* calculate total refresh cost */ + for (unsigned int i = 0; ifee_withdraw); + TALER_amount_ntoh (&value, + &new_issues[i]->value); + if ( (GNUNET_OK != + TALER_amount_add (&refresh_cost, + &refresh_cost, + &fee)) || + (GNUNET_OK != + TALER_amount_add (&refresh_cost, + &refresh_cost, + &value)) ) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + } + + /* compute contribution of old coin */ + { + struct TALER_Amount melt_fee; + + TALER_amount_ntoh (&melt_fee, + &issue->fee_refresh); + if (GNUNET_OK != + TALER_amount_subtract (&amount_without_fee, + amount_with_fee, + &melt_fee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + } + + /* check old coin covers complete expenses */ + if (1 == TALER_amount_cmp (&refresh_cost, + &amount_without_fee)) + { + /* refresh_cost > amount_without_fee */ + report_amount_arithmetic_inconsistency ("melt (fee)", + rowid, + &amount_without_fee, + &refresh_cost, + -1); + return GNUNET_OK; + } + + /* update outstanding denomination amounts */ + for (unsigned int i = 0; idenom_hash); + if (NULL == dsi) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + TALER_amount_ntoh (&value, + &new_issues[i]->value); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Created fresh coin in denomination `%s' of value %s\n", + GNUNET_h2s (&new_issues[i]->denom_hash), + TALER_amount2s (&value)); + dsi->num_issued++; + if (GNUNET_OK != + TALER_amount_add (&dsi->denom_balance, + &dsi->denom_balance, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&dsi->denom_risk, + &dsi->denom_risk, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "New balance of denomination `%s' is %s\n", + GNUNET_h2s (&new_issues[i]->denom_hash), + TALER_amount2s (&dsi->denom_balance)); + if (GNUNET_OK != + TALER_amount_add (&total_escrow_balance, + &total_escrow_balance, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&total_risk, + &total_risk, + &value)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + } + } + } + + /* update old coin's denomination balance */ + dso = get_denomination_summary (cc, + issue, + &issue->denom_hash); + if (NULL == dso) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (GNUNET_SYSERR == + TALER_amount_subtract (&tmp, + &dso->denom_balance, + amount_with_fee)) + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&dso->denom_loss, + &dso->denom_loss, + amount_with_fee)); + dso->report_emergency = GNUNET_YES; + } + else + { + dso->denom_balance = tmp; + } + if (-1 == TALER_amount_cmp (&total_escrow_balance, + amount_with_fee)) + { + /* This can theoretically happen if for example the exchange + never issued any coins (i.e. escrow balance is zero), but + accepted a forged coin (i.e. emergency situation after + private key compromise). In that case, we cannot even + subtract the profit we make from the fee from the escrow + balance. Tested as part of test-auditor.sh, case #18 */report_amount_arithmetic_inconsistency ( + "subtracting refresh fee from escrow balance", + rowid, + &total_escrow_balance, + amount_with_fee, + 0); + } + else + { + GNUNET_assert (GNUNET_SYSERR != + TALER_amount_subtract (&total_escrow_balance, + &total_escrow_balance, + amount_with_fee)); + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "New balance of denomination `%s' after melt is %s\n", + GNUNET_h2s (&issue->denom_hash), + TALER_amount2s (&dso->denom_balance)); + + /* update global melt fees */ + { + struct TALER_Amount rfee; + + TALER_amount_ntoh (&rfee, + &issue->fee_refresh); + if (GNUNET_OK != + TALER_amount_add (&total_melt_fee_income, + &total_melt_fee_income, + &rfee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + } + + /* We're good! */ + return GNUNET_OK; +} + + +/** + * Function called with details about deposits that have been made, + * with the goal of auditing the deposit's execution. + * + * @param cls closure + * @param rowid unique serial ID for the deposit in our DB + * @param timestamp when did the deposit happen + * @param merchant_pub public key of the merchant + * @param denom_pub denomination public key of @a coin_pub + * @param coin_pub public key of the coin + * @param coin_sig signature from the coin + * @param amount_with_fee amount that was deposited including fee + * @param h_contract_terms hash of the proposal data known to merchant and customer + * @param refund_deadline by which the merchant adviced that he might want + * to get a refund + * @param wire_deadline by which the merchant adviced that he would like the + * wire transfer to be executed + * @param receiver_wire_account wire details for the merchant, NULL from iterate_matching_deposits() + * @param done flag set if the deposit was already executed (or not) + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +deposit_cb (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute timestamp, + const struct TALER_MerchantPublicKeyP *merchant_pub, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_CoinSpendSignatureP *coin_sig, + const struct TALER_Amount *amount_with_fee, + const struct GNUNET_HashCode *h_contract_terms, + struct GNUNET_TIME_Absolute refund_deadline, + struct GNUNET_TIME_Absolute wire_deadline, + const json_t *receiver_wire_account, + int done) +{ + struct CoinContext *cc = cls; + const struct TALER_DenominationKeyValidityPS *issue; + struct DenominationSummary *ds; + struct TALER_DepositRequestPS dr; + struct TALER_Amount tmp; + enum GNUNET_DB_QueryStatus qs; + + (void) wire_deadline; + (void) done; + GNUNET_assert (rowid >= ppc.last_deposit_serial_id); /* should be monotonically increasing */ + ppc.last_deposit_serial_id = rowid + 1; + + qs = TALER_ARL_get_denomination_info (denom_pub, + &issue, + NULL); + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("deposits", + rowid, + "denomination key not found"); + return GNUNET_OK; + } + + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + return GNUNET_SYSERR; + } + qs = check_known_coin (coin_pub, + denom_pub, + amount_with_fee); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + return GNUNET_SYSERR; + } + + /* Verify deposit signature */ + dr.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_DEPOSIT); + dr.purpose.size = htonl (sizeof (dr)); + dr.h_contract_terms = *h_contract_terms; + if (GNUNET_OK != + TALER_JSON_merchant_wire_signature_hash (receiver_wire_account, + &dr.h_wire)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "deposit", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount ( + amount_with_fee), + "key_pub", GNUNET_JSON_from_data_auto ( + coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount_with_fee)); + return GNUNET_OK; + } + dr.timestamp = GNUNET_TIME_absolute_hton (timestamp); + dr.refund_deadline = GNUNET_TIME_absolute_hton (refund_deadline); + TALER_amount_hton (&dr.amount_with_fee, + amount_with_fee); + dr.deposit_fee = issue->fee_deposit; + dr.merchant = *merchant_pub; + dr.coin_pub = *coin_pub; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_DEPOSIT, + &dr.purpose, + &coin_sig->eddsa_signature, + &coin_pub->eddsa_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "deposit", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount ( + amount_with_fee), + "key_pub", GNUNET_JSON_from_data_auto ( + coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount_with_fee)); + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Deposited coin %s in denomination `%s' of value %s\n", + TALER_B2S (coin_pub), + GNUNET_h2s (&issue->denom_hash), + TALER_amount2s (amount_with_fee)); + + /* update old coin's denomination balance */ + ds = get_denomination_summary (cc, + issue, + &issue->denom_hash); + if (NULL == ds) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (GNUNET_SYSERR == + TALER_amount_subtract (&tmp, + &ds->denom_balance, + amount_with_fee)) + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&ds->denom_loss, + &ds->denom_loss, + amount_with_fee)); + ds->report_emergency = GNUNET_YES; + } + else + { + ds->denom_balance = tmp; + } + + if (-1 == TALER_amount_cmp (&total_escrow_balance, + amount_with_fee)) + { + /* This can theoretically happen if for example the exchange + never issued any coins (i.e. escrow balance is zero), but + accepted a forged coin (i.e. emergency situation after + private key compromise). In that case, we cannot even + subtract the profit we make from the fee from the escrow + balance. Tested as part of test-auditor.sh, case #18 */report_amount_arithmetic_inconsistency ( + "subtracting deposit fee from escrow balance", + rowid, + &total_escrow_balance, + amount_with_fee, + 0); + } + else + { + GNUNET_assert (GNUNET_SYSERR != + TALER_amount_subtract (&total_escrow_balance, + &total_escrow_balance, + amount_with_fee)); + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "New balance of denomination `%s' after deposit is %s\n", + GNUNET_h2s (&issue->denom_hash), + TALER_amount2s (&ds->denom_balance)); + + /* update global up melt fees */ + { + struct TALER_Amount dfee; + + TALER_amount_ntoh (&dfee, + &issue->fee_deposit); + if (GNUNET_OK != + TALER_amount_add (&total_deposit_fee_income, + &total_deposit_fee_income, + &dfee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + } + + return GNUNET_OK; +} + + +/** + * Function called with details about coins that were refunding, + * with the goal of auditing the refund's execution. Adds the + * refunded amount back to the outstanding balance of the respective + * denomination. + * + * @param cls closure + * @param rowid unique serial ID for the refund in our DB + * @param denom_pub denomination public key of @a coin_pub + * @param coin_pub public key of the coin + * @param merchant_pub public key of the merchant + * @param merchant_sig signature of the merchant + * @param h_contract_terms hash of the proposal data known to merchant and customer + * @param rtransaction_id refund transaction ID chosen by the merchant + * @param amount_with_fee amount that was deposited including fee + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +refund_cb (void *cls, + uint64_t rowid, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_MerchantPublicKeyP *merchant_pub, + const struct TALER_MerchantSignatureP *merchant_sig, + const struct GNUNET_HashCode *h_contract_terms, + uint64_t rtransaction_id, + const struct TALER_Amount *amount_with_fee) +{ + struct CoinContext *cc = cls; + const struct TALER_DenominationKeyValidityPS *issue; + struct DenominationSummary *ds; + struct TALER_RefundRequestPS rr; + struct TALER_Amount amount_without_fee; + struct TALER_Amount refund_fee; + enum GNUNET_DB_QueryStatus qs; + + GNUNET_assert (rowid >= ppc.last_refund_serial_id); /* should be monotonically increasing */ + ppc.last_refund_serial_id = rowid + 1; + + qs = TALER_ARL_get_denomination_info (denom_pub, + &issue, + NULL); + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("refunds", + rowid, + "denomination key not found"); + return GNUNET_SYSERR; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return GNUNET_SYSERR; + } + + /* verify refund signature */ + rr.purpose.purpose = htonl (TALER_SIGNATURE_MERCHANT_REFUND); + rr.purpose.size = htonl (sizeof (rr)); + rr.h_contract_terms = *h_contract_terms; + rr.coin_pub = *coin_pub; + rr.merchant = *merchant_pub; + rr.rtransaction_id = GNUNET_htonll (rtransaction_id); + TALER_amount_hton (&rr.refund_amount, + amount_with_fee); + rr.refund_fee = issue->fee_refund; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_MERCHANT_REFUND, + &rr.purpose, + &merchant_sig->eddsa_sig, + &merchant_pub->eddsa_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "refund", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount ( + amount_with_fee), + "key_pub", GNUNET_JSON_from_data_auto ( + merchant_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount_with_fee)); + return GNUNET_OK; + } + + TALER_amount_ntoh (&refund_fee, + &issue->fee_refund); + if (GNUNET_OK != + TALER_amount_subtract (&amount_without_fee, + amount_with_fee, + &refund_fee)) + { + report_amount_arithmetic_inconsistency ("refund (fee)", + rowid, + &amount_without_fee, + &refund_fee, + -1); + return GNUNET_OK; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Refunding coin %s in denomination `%s' value %s\n", + TALER_B2S (coin_pub), + GNUNET_h2s (&issue->denom_hash), + TALER_amount2s (amount_with_fee)); + + /* update coin's denomination balance */ + ds = get_denomination_summary (cc, + issue, + &issue->denom_hash); + if (NULL == ds) + { + GNUNET_break (0); + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&ds->denom_balance, + &ds->denom_balance, + &amount_without_fee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&ds->denom_risk, + &ds->denom_risk, + &amount_without_fee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&total_escrow_balance, + &total_escrow_balance, + &amount_without_fee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + if (GNUNET_OK != + TALER_amount_add (&total_risk, + &total_risk, + &amount_without_fee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "New balance of denomination `%s' after refund is %s\n", + GNUNET_h2s (&issue->denom_hash), + TALER_amount2s (&ds->denom_balance)); + + /* update total refund fee balance */ + if (GNUNET_OK != + TALER_amount_add (&total_refund_fee_income, + &total_refund_fee_income, + &refund_fee)) + { + GNUNET_break (0); + cc->qs = GNUNET_DB_STATUS_HARD_ERROR; + return GNUNET_SYSERR; + } + + return GNUNET_OK; +} + + +/** + * Check that the recoup operation was properly initiated by a coin + * and update the denomination's losses accordingly. + * + * @param cc the context with details about the coin + * @param rowid row identifier used to uniquely identify the recoup operation + * @param amount how much should be added back to the reserve + * @param coin public information about the coin + * @param denom_pub public key of the denomionation of @a coin + * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP + * @param coin_blind blinding factor used to blind the coin + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +check_recoup (struct CoinContext *cc, + uint64_t rowid, + const struct TALER_Amount *amount, + const struct TALER_CoinPublicInfo *coin, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendSignatureP *coin_sig, + const struct TALER_DenominationBlindingKeyP *coin_blind) +{ + struct TALER_RecoupRequestPS pr; + struct DenominationSummary *ds; + enum GNUNET_DB_QueryStatus qs; + const struct TALER_DenominationKeyValidityPS *issue; + + if (GNUNET_OK != + TALER_test_coin_valid (coin, + denom_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "recoup", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount (amount), + "key_pub", GNUNET_JSON_from_data_auto ( + &pr.h_denom_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount)); + } + qs = TALER_ARL_get_denomination_info (denom_pub, + &issue, + &pr.h_denom_pub); + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("recoup", + rowid, + "denomination key not found"); + return GNUNET_OK; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + /* The key not existing should be prevented by foreign key constraints, + so must be a transient DB error. */ + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + cc->qs = qs; + return GNUNET_SYSERR; + } + pr.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_RECOUP); + pr.purpose.size = htonl (sizeof (pr)); + pr.coin_pub = coin->coin_pub; + pr.coin_blind = *coin_blind; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_RECOUP, + &pr.purpose, + &coin_sig->eddsa_signature, + &coin->coin_pub.eddsa_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "recoup", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount (amount), + "coin_pub", GNUNET_JSON_from_data_auto ( + &coin->coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount)); + return GNUNET_OK; + } + ds = get_denomination_summary (cc, + issue, + &issue->denom_hash); + if (GNUNET_NO == ds->was_revoked) + { + /* Woopsie, we allowed recoup on non-revoked denomination!? */ + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", + "recoup (denomination not revoked)", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount (amount), + "coin_pub", GNUNET_JSON_from_data_auto ( + &coin->coin_pub))); + } + GNUNET_break (GNUNET_OK == + TALER_amount_add (&ds->denom_recoup, + &ds->denom_recoup, + amount)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_recoup_loss, + &total_recoup_loss, + amount)); + return GNUNET_OK; +} + + +/** + * Function called about recoups the exchange has to perform. + * + * @param cls a `struct CoinContext *` + * @param rowid row identifier used to uniquely identify the recoup operation + * @param timestamp when did we receive the recoup request + * @param amount how much should be added back to the reserve + * @param reserve_pub public key of the reserve + * @param coin public information about the coin + * @param denom_pub denomination public key of @a coin + * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP + * @param coin_blind blinding factor used to blind the coin + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +recoup_cb (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute timestamp, + const struct TALER_Amount *amount, + const struct TALER_ReservePublicKeyP *reserve_pub, + const struct TALER_CoinPublicInfo *coin, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendSignatureP *coin_sig, + const struct TALER_DenominationBlindingKeyP *coin_blind) +{ + struct CoinContext *cc = cls; + + (void) timestamp; + (void) reserve_pub; + return check_recoup (cc, + rowid, + amount, + coin, + denom_pub, + coin_sig, + coin_blind); +} + + +/** + * Function called about recoups on refreshed coins the exchange has to + * perform. + * + * @param cls a `struct CoinContext *` + * @param rowid row identifier used to uniquely identify the recoup operation + * @param timestamp when did we receive the recoup request + * @param amount how much should be added back to the reserve + * @param old_coin_pub original coin that was refreshed to create @a coin + * @param coin public information about the coin + * @param denom_pub denomination public key of @a coin + * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP + * @param coin_blind blinding factor used to blind the coin + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +recoup_refresh_cb (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute timestamp, + const struct TALER_Amount *amount, + const struct TALER_CoinSpendPublicKeyP *old_coin_pub, + const struct TALER_CoinPublicInfo *coin, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendSignatureP *coin_sig, + const struct TALER_DenominationBlindingKeyP *coin_blind) +{ + struct CoinContext *cc = cls; + + (void) timestamp; + (void) old_coin_pub; + return check_recoup (cc, + rowid, + amount, + coin, + denom_pub, + coin_sig, + coin_blind); +} + + +/** + * Analyze the exchange's processing of coins. + * + * @param cls closure + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +analyze_coins (void *cls) +{ + struct CoinContext cc; + enum GNUNET_DB_QueryStatus qs; + enum GNUNET_DB_QueryStatus qsx; + enum GNUNET_DB_QueryStatus qsp; + + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Analyzing coins\n"); + qsp = TALER_ARL_adb->get_auditor_progress_coin (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppc); + if (0 > qsp) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); + return qsp; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) + { + GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, + "First analysis using this auditor, starting from scratch\n"); + } + else + { + ppc_start = ppc; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Resuming coin audit at %llu/%llu/%llu/%llu/%llu\n", + (unsigned long long) ppc.last_deposit_serial_id, + (unsigned long long) ppc.last_melt_serial_id, + (unsigned long long) ppc.last_refund_serial_id, + (unsigned long long) ppc.last_withdraw_serial_id, + (unsigned long long) ppc.last_recoup_refresh_serial_id); + } + + /* setup 'cc' */ + cc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + cc.denom_summaries = GNUNET_CONTAINER_multihashmap_create (256, + GNUNET_NO); + qsx = TALER_ARL_adb->get_balance_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_escrow_balance, + &total_deposit_fee_income, + &total_melt_fee_income, + &total_refund_fee_income, + &total_risk, + &total_recoup_loss, + &total_irregular_recoups); + if (0 > qsx) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); + return qsx; + } + + /* process withdrawals */ + if (0 > + (qs = TALER_ARL_edb->select_withdrawals_above_serial_id ( + TALER_ARL_edb->cls, + TALER_ARL_esession, + ppc. + last_withdraw_serial_id, + &withdraw_cb, + &cc)) ) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 > cc.qs) + return cc.qs; + + /* process refunds */ + if (0 > + (qs = TALER_ARL_edb->select_refunds_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppc. + last_refund_serial_id, + &refund_cb, + &cc))) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 > cc.qs) + return cc.qs; + + /* process refreshs */ + if (0 > + (qs = TALER_ARL_edb->select_refreshes_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppc. + last_melt_serial_id, + &refresh_session_cb, + &cc))) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 > cc.qs) + return cc.qs; + + /* process deposits */ + if (0 > + (qs = TALER_ARL_edb->select_deposits_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppc. + last_deposit_serial_id, + &deposit_cb, + &cc))) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 > cc.qs) + return cc.qs; + + /* process recoups */ + if (0 > + (qs = TALER_ARL_edb->select_recoup_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppc. + last_recoup_serial_id, + &recoup_cb, + &cc))) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 > cc.qs) + return cc.qs; + if (0 > + (qs = TALER_ARL_edb->select_recoup_refresh_above_serial_id ( + TALER_ARL_edb->cls, + TALER_ARL_esession, + ppc. + last_recoup_refresh_serial_id, + & + recoup_refresh_cb, + &cc))) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (0 > cc.qs) + return cc.qs; + + /* sync 'cc' back to disk */ + cc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + GNUNET_CONTAINER_multihashmap_iterate (cc.denom_summaries, + &sync_denomination, + &cc); + GNUNET_CONTAINER_multihashmap_destroy (cc.denom_summaries); + if (0 > cc.qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == cc.qs); + return cc.qs; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsx) + qs = TALER_ARL_adb->update_balance_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_escrow_balance, + &total_deposit_fee_income, + &total_melt_fee_income, + &total_refund_fee_income, + &total_risk, + &total_recoup_loss, + &total_irregular_recoups); + else + qs = TALER_ARL_adb->insert_balance_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_escrow_balance, + &total_deposit_fee_income, + &total_melt_fee_income, + &total_refund_fee_income, + &total_risk, + &total_recoup_loss, + &total_irregular_recoups); + if (0 >= qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) + qs = TALER_ARL_adb->update_auditor_progress_coin (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppc); + else + qs = TALER_ARL_adb->insert_auditor_progress_coin (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppc); + if (0 >= qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to update auditor DB, not recording progress\n"); + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Concluded coin audit step at %llu/%llu/%llu/%llu/%llu\n"), + (unsigned long long) ppc.last_deposit_serial_id, + (unsigned long long) ppc.last_melt_serial_id, + (unsigned long long) ppc.last_refund_serial_id, + (unsigned long long) ppc.last_withdraw_serial_id, + (unsigned long long) ppc.last_recoup_refresh_serial_id); + return qs; +} + + +/** + * Main function that will be run. + * + * @param cls closure + * @param args remaining command-line arguments + * @param TALER_ARL_cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *TALER_ARL_cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + json_t *report; + + (void) cls; + (void) args; + (void) TALER_ARL_cfgfile; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Launching auditor\n"); + if (GNUNET_OK != + TALER_ARL_init (c)) + { + global_ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting audit\n"); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &reported_emergency_loss)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + & + reported_emergency_risk_by_amount)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + & + reported_emergency_risk_by_count)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + & + reported_emergency_loss_by_count)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_escrow_balance)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_risk)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_recoup_loss)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_irregular_recoups)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_deposit_fee_income)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_melt_fee_income)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_refund_fee_income)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_arithmetic_delta_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_arithmetic_delta_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_sig_loss)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_refresh_hanging)); + GNUNET_assert (NULL != + (report_emergencies = json_array ())); + GNUNET_assert (NULL != + (report_emergencies_by_count = json_array ())); + GNUNET_assert (NULL != + (report_row_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_amount_arithmetic_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_bad_sig_losses = json_array ())); + GNUNET_assert (NULL != + (report_refreshs_hanging = json_array ())); + if (GNUNET_OK != + TALER_ARL_setup_sessions_and_run (&analyze_coins, + NULL)) + { + global_ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Audit complete\n"); + report = json_pack ("{s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:I, s:I, s:I, s:I, s:I," + " s:I, s:I, s:I, s:I, s:I," + " s:I, s:I, s:o, s:o, s:o}", + /* Block #1 */ + "total_escrow_balance", + TALER_JSON_from_amount (&total_escrow_balance), + "total_active_risk", + TALER_JSON_from_amount (&total_risk), + "total_deposit_fee_income", + TALER_JSON_from_amount ( + &total_deposit_fee_income), + "total_melt_fee_income", + TALER_JSON_from_amount (&total_melt_fee_income), + "total_refund_fee_income", + TALER_JSON_from_amount ( + &total_refund_fee_income), + /* Block #2 */ + /* Tested in test-auditor.sh #18 */ + "emergencies", + report_emergencies, + /* Tested in test-auditor.sh #18 */ + "emergencies_risk_by_amount", + TALER_JSON_from_amount ( + &reported_emergency_risk_by_amount), + /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ + "bad_sig_losses", + report_bad_sig_losses, + /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ + "total_bad_sig_loss", + TALER_JSON_from_amount (&total_bad_sig_loss), + /* Tested in test-auditor.sh #14/#15 */ + "row_inconsistencies", + report_row_inconsistencies, + /* Block #3 */ + "amount_arithmetic_inconsistencies", + report_amount_arithmetic_inconsistencies, + "total_arithmetic_delta_plus", + TALER_JSON_from_amount ( + &total_arithmetic_delta_plus), + "total_arithmetic_delta_minus", + TALER_JSON_from_amount ( + &total_arithmetic_delta_minus), + /* Tested in test-auditor.sh #12 */ + "total_refresh_hanging", + TALER_JSON_from_amount (&total_refresh_hanging), + /* Tested in test-auditor.sh #12 */ + "refresh_hanging", + report_refreshs_hanging, + /* Block #4 */ + "total_recoup_loss", + TALER_JSON_from_amount (&total_recoup_loss), + /* Tested in test-auditor.sh #18 */ + "emergencies_by_count", + report_emergencies_by_count, + /* Tested in test-auditor.sh #18 */ + "emergencies_risk_by_count", + TALER_JSON_from_amount ( + &reported_emergency_risk_by_count), + /* Tested in test-auditor.sh #18 */ + "emergencies_loss", + TALER_JSON_from_amount ( + &reported_emergency_loss), + /* Tested in test-auditor.sh #18 */ + "emergencies_loss_by_count", + TALER_JSON_from_amount ( + &reported_emergency_loss_by_count), + /* Block #5 */ + "start_ppc_withdraw_serial_id", + (json_int_t) ppc_start.last_withdraw_serial_id, + "start_ppc_deposit_serial_id", + (json_int_t) ppc_start.last_deposit_serial_id, + "start_ppc_melt_serial_id", + (json_int_t) ppc_start.last_melt_serial_id, + "start_ppc_refund_serial_id", + (json_int_t) ppc_start.last_refund_serial_id, + "start_ppc_recoup_serial_id", + (json_int_t) ppc_start.last_recoup_serial_id, + /* Block #6 */ + "start_ppc_recoup_refresh_serial_id", + (json_int_t) ppc_start. + last_recoup_refresh_serial_id, + "end_ppc_withdraw_serial_id", + (json_int_t) ppc.last_withdraw_serial_id, + "end_ppc_deposit_serial_id", + (json_int_t) ppc.last_deposit_serial_id, + "end_ppc_melt_serial_id", + (json_int_t) ppc.last_melt_serial_id, + "end_ppc_refund_serial_id", + (json_int_t) ppc.last_refund_serial_id, + /* Block #7 */ + "end_ppc_recoup_serial_id", + (json_int_t) ppc.last_recoup_serial_id, + "end_ppc_recoup_refresh_serial_id", + (json_int_t) ppc.last_recoup_refresh_serial_id, + "auditor_start_time", json_string ( + GNUNET_STRINGS_absolute_time_to_string ( + start_time)), + "auditor_end_time", json_string ( + GNUNET_STRINGS_absolute_time_to_string ( + GNUNET_TIME_absolute_get ())), + "total_irregular_recoups", + TALER_JSON_from_amount ( + &total_irregular_recoups) + ); + GNUNET_break (NULL != report); + TALER_ARL_done (report); +} + + +/** + * The main function of the database initialization tool. + * Used to initialize the Taler Exchange's database. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + const struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_base32_auto ('m', + "exchange-key", + "KEY", + "public key of the exchange (Crockford base32 encoded)", + &TALER_ARL_master_pub), + GNUNET_GETOPT_option_flag ('r', + "TALER_ARL_restart", + "TALER_ARL_restart audit from the beginning (required on first run)", + &TALER_ARL_restart), + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_OPTION_END + }; + + /* force linker to link against libtalerutil; if we do + not do this, the linker may "optimize" libtalerutil + away and skip #TALER_OS_init(), which we do need */ + (void) TALER_project_data_default (); + GNUNET_assert (GNUNET_OK == + GNUNET_log_setup ("taler-auditor", + "MESSAGE", + NULL)); + if (GNUNET_OK != + GNUNET_PROGRAM_run (argc, + argv, + "taler-auditor", + "Audit Taler exchange database", + options, + &run, + NULL)) + return 1; + return global_ret; +} + + +/* end of taler-auditor.c */ diff --git a/src/auditor/taler-helper-auditor-deposits.c b/src/auditor/taler-helper-auditor-deposits.c new file mode 100644 index 000000000..eb3e0e7a3 --- /dev/null +++ b/src/auditor/taler-helper-auditor-deposits.c @@ -0,0 +1,369 @@ +/* + This file is part of TALER + Copyright (C) 2016-2020 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. + + You should have received a copy of the GNU Affero Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file auditor/taler-auditor-deposits.c + * @brief audits an exchange database for deposit confirmation consistency + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include "taler_auditordb_plugin.h" +#include "taler_exchangedb_lib.h" +#include "taler_json_lib.h" +#include "taler_bank_service.h" +#include "taler_signatures.h" +#include "report-lib.h" + + +/** + * Return value from main(). + */ +static int global_ret; + +/** + * Array of TALER_ARL_reports about missing deposit confirmations. + */ +static json_t *report_deposit_confirmation_inconsistencies; + +/** + * Total number of deposit confirmations that we did not get. + */ +static json_int_t number_missed_deposit_confirmations; + +/** + * Total amount involved in deposit confirmations that we did not get. + */ +static struct TALER_Amount total_missed_deposit_confirmations; + + +/* *************************** Analysis of deposit-confirmations ********** */ + +/** + * Closure for #test_dc. + */ +struct DepositConfirmationContext +{ + + /** + * How many deposit confirmations did we NOT find in the #TALER_ARL_edb? + */ + unsigned long long missed_count; + + /** + * What is the total amount missing? + */ + struct TALER_Amount missed_amount; + + /** + * Lowest SerialID of the first coin we missed? (This is where we + * should resume next time). + */ + uint64_t first_missed_coin_serial; + + /** + * Lowest SerialID of the first coin we missed? (This is where we + * should resume next time). + */ + uint64_t last_seen_coin_serial; + + /** + * Success or failure of (exchange) database operations within + * #test_dc. + */ + enum GNUNET_DB_QueryStatus qs; + +}; + + +/** + * Given a deposit confirmation from #TALER_ARL_adb, check that it is also + * in #TALER_ARL_edb. Update the deposit confirmation context accordingly. + * + * @param cls our `struct DepositConfirmationContext` + * @param serial_id row of the @a dc in the database + * @param dc the deposit confirmation we know + */ +static void +test_dc (void *cls, + uint64_t serial_id, + const struct TALER_AUDITORDB_DepositConfirmation *dc) +{ + struct DepositConfirmationContext *dcc = cls; + enum GNUNET_DB_QueryStatus qs; + struct TALER_EXCHANGEDB_Deposit dep; + + dcc->last_seen_coin_serial = serial_id; + memset (&dep, + 0, + sizeof (dep)); + dep.coin.coin_pub = dc->coin_pub; + dep.h_contract_terms = dc->h_contract_terms; + dep.merchant_pub = dc->merchant; + dep.h_wire = dc->h_wire; + dep.refund_deadline = dc->refund_deadline; + + qs = TALER_ARL_edb->have_deposit (TALER_ARL_edb->cls, + TALER_ARL_esession, + &dep, + GNUNET_NO /* do not check refund deadline */); + if (qs > 0) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Found deposit %s in exchange database\n", + GNUNET_h2s (&dc->h_contract_terms)); + return; /* found, all good */ + } + if (qs < 0) + { + GNUNET_break (0); /* DB error, complain */ + dcc->qs = qs; + return; + } + /* deposit confirmation missing! TALER_ARL_report! */ + TALER_ARL_report (report_deposit_confirmation_inconsistencies, + json_pack ("{s:o, s:o, s:I, s:o}", + "timestamp", + TALER_ARL_json_from_time_abs (dc->timestamp), + "amount", + TALER_JSON_from_amount (&dc->amount_without_fee), + "rowid", + (json_int_t) serial_id, + "account", + GNUNET_JSON_from_data_auto (&dc->h_wire))); + dcc->first_missed_coin_serial = GNUNET_MIN (dcc->first_missed_coin_serial, + serial_id); + dcc->missed_count++; + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&dcc->missed_amount, + &dcc->missed_amount, + &dc->amount_without_fee)); +} + + +/** + * Check that the deposit-confirmations that were TALER_ARL_reported to + * us by merchants are also in the exchange's database. + * + * @param cls closure + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +analyze_deposit_confirmations (void *cls) +{ + struct TALER_AUDITORDB_ProgressPointDepositConfirmation ppdc; + struct DepositConfirmationContext dcc; + enum GNUNET_DB_QueryStatus qs; + enum GNUNET_DB_QueryStatus qsx; + enum GNUNET_DB_QueryStatus qsp; + + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing deposit confirmations\n"); + ppdc.last_deposit_confirmation_serial_id = 0; + qsp = TALER_ARL_adb->get_auditor_progress_deposit_confirmation ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + &ppdc); + if (0 > qsp) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); + return qsp; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) + { + GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, + _ ( + "First analysis using this auditor, starting audit from scratch\n")); + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Resuming deposit confirmation audit at %llu\n"), + (unsigned long long) ppdc.last_deposit_confirmation_serial_id); + } + + /* setup 'cc' */ + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &dcc.missed_amount)); + dcc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + dcc.missed_count = 0LLU; + dcc.first_missed_coin_serial = UINT64_MAX; + qsx = TALER_ARL_adb->get_deposit_confirmations (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + ppdc. + last_deposit_confirmation_serial_id, + &test_dc, + &dcc); + if (0 > qsx) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); + return qsx; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzed %d deposit confirmations (above serial ID %llu)\n", + (int) qsx, + (unsigned long long) ppdc.last_deposit_confirmation_serial_id); + if (0 > dcc.qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == dcc.qs); + return dcc.qs; + } + if (UINT64_MAX == dcc.first_missed_coin_serial) + ppdc.last_deposit_confirmation_serial_id = dcc.last_seen_coin_serial; + else + ppdc.last_deposit_confirmation_serial_id = dcc.first_missed_coin_serial - 1; + + /* sync 'cc' back to disk */ + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) + qs = TALER_ARL_adb->update_auditor_progress_deposit_confirmation ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + &ppdc); + else + qs = TALER_ARL_adb->insert_auditor_progress_deposit_confirmation ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + &ppdc); + if (0 >= qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to update auditor DB, not recording progress\n"); + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + number_missed_deposit_confirmations = (json_int_t) dcc.missed_count; + total_missed_deposit_confirmations = dcc.missed_amount; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Concluded deposit confirmation audit step at %llu\n"), + (unsigned long long) ppdc.last_deposit_confirmation_serial_id); + return qs; +} + + +/** + * Main function that will be run. + * + * @param cls closure + * @param args remaining command-line arguments + * @param TALER_ARL_cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *TALER_ARL_cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + (void) cls; + (void) args; + (void) TALER_ARL_cfgfile; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Launching auditor\n"); + if (GNUNET_OK != + TALER_ARL_init (c)) + { + global_ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting audit\n"); + GNUNET_assert (NULL != + (report_deposit_confirmation_inconsistencies = json_array ())); + if (GNUNET_OK != + TALER_ARL_setup_sessions_and_run (&analyze_deposit_confirmations, + NULL)) + { + global_ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Audit complete\n"); + { + json_t *report; + + report = json_pack ("{s:o, s:o, s:I}", + "deposit_confirmation_inconsistencies", + report_deposit_confirmation_inconsistencies, + "missing_deposit_confirmation_count", + (json_int_t) number_missed_deposit_confirmations, + "missing_deposit_confirmation_total", + TALER_JSON_from_amount ( + &total_missed_deposit_confirmations) + ); + GNUNET_break (NULL != report); + TALER_ARL_done (report); + } +} + + +/** + * The main function of the database initialization tool. + * Used to initialize the Taler Exchange's database. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + const struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_base32_auto ('m', + "exchange-key", + "KEY", + "public key of the exchange (Crockford base32 encoded)", + &TALER_ARL_master_pub), + GNUNET_GETOPT_option_flag ('r', + "TALER_ARL_restart", + "TALER_ARL_restart audit from the beginning (required on first run)", + &TALER_ARL_restart), + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_OPTION_END + }; + + /* force linker to link against libtalerutil; if we do + not do this, the linker may "optimize" libtalerutil + away and skip #TALER_OS_init(), which we do need */ + (void) TALER_project_data_default (); + GNUNET_assert (GNUNET_OK == + GNUNET_log_setup ("taler-auditor-deposits", + "MESSAGE", + NULL)); + if (GNUNET_OK != + GNUNET_PROGRAM_run (argc, + argv, + "taler-auditor-deposits", + "Audit Taler exchange database for deposit confirmation consistency", + options, + &run, + NULL)) + return 1; + return global_ret; +} + + +/* end of taler-auditor-deposits.c */ diff --git a/src/auditor/taler-helper-auditor-reserves.c b/src/auditor/taler-helper-auditor-reserves.c new file mode 100644 index 000000000..cd0f1b98b --- /dev/null +++ b/src/auditor/taler-helper-auditor-reserves.c @@ -0,0 +1,1674 @@ +/* + This file is part of TALER + Copyright (C) 2016-2020 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU Affero Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU Affero Public License for more details. + + You should have received a copy of the GNU Affero Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file auditor/taler-helper-auditor-reserves.c + * @brief audits the reserves of an exchange database + * @author Christian Grothoff + */ +#include "platform.h" +#include +#include "taler_auditordb_plugin.h" +#include "taler_exchangedb_lib.h" +#include "taler_json_lib.h" +#include "taler_bank_service.h" +#include "taler_signatures.h" +#include "report-lib.h" + + +/** + * Use a 1 day grace period to deal with clocks not being perfectly synchronized. + */ +#define CLOSING_GRACE_PERIOD GNUNET_TIME_UNIT_DAYS + +/** + * Return value from main(). + */ +static int global_ret; + +/** + * After how long should idle reserves be closed? + */ +static struct GNUNET_TIME_Relative idle_reserve_expiration_time; + +/** + * Checkpointing our progress for reserves. + */ +static struct TALER_AUDITORDB_ProgressPointReserve ppr; + +/** + * Checkpointing our progress for reserves. + */ +static struct TALER_AUDITORDB_ProgressPointReserve ppr_start; + +/** + * Array of reports about row inconsitencies. + */ +static json_t *report_row_inconsistencies; + +/** + * Array of reports about the denomination key not being + * valid at the time of withdrawal. + */ +static json_t *denomination_key_validity_withdraw_inconsistencies; + +/** + * Array of reports about reserve balance insufficient inconsitencies. + */ +static json_t *report_reserve_balance_insufficient_inconsistencies; + +/** + * Total amount reserves were charged beyond their balance. + */ +static struct TALER_Amount total_balance_insufficient_loss; + +/** + * Array of reports about reserve balance summary wrong in database. + */ +static json_t *report_reserve_balance_summary_wrong_inconsistencies; + +/** + * Total delta between expected and stored reserve balance summaries, + * for positive deltas. + */ +static struct TALER_Amount total_balance_summary_delta_plus; + +/** + * Total delta between expected and stored reserve balance summaries, + * for negative deltas. + */ +static struct TALER_Amount total_balance_summary_delta_minus; + +/** + * Array of reports about reserve's not being closed inconsitencies. + */ +static json_t *report_reserve_not_closed_inconsistencies; + +/** + * Total amount affected by reserves not having been closed on time. + */ +static struct TALER_Amount total_balance_reserve_not_closed; + +/** + * Report about amount calculation differences (causing profit + * or loss at the exchange). + */ +static json_t *report_amount_arithmetic_inconsistencies; + +/** + * Profits the exchange made by bad amount calculations. + */ +static struct TALER_Amount total_arithmetic_delta_plus; + +/** + * Losses the exchange made by bad amount calculations. + */ +static struct TALER_Amount total_arithmetic_delta_minus; + +/** + * Expected balance in the escrow account. + */ +static struct TALER_Amount total_escrow_balance; + +/** + * Recoups we made on denominations that were not revoked (!?). + */ +static struct TALER_Amount total_irregular_recoups; + +/** + * Total withdraw fees earned. + */ +static struct TALER_Amount total_withdraw_fee_income; + +/** + * Array of reports about coin operations with bad signatures. + */ +static json_t *report_bad_sig_losses; + +/** + * Total amount lost by operations for which signatures were invalid. + */ +static struct TALER_Amount total_bad_sig_loss; + + +/* ***************************** Report logic **************************** */ + + +/** + * Report a (serious) inconsistency in the exchange's database with + * respect to calculations involving amounts. + * + * @param operation what operation had the inconsistency + * @param rowid affected row, UINT64_MAX if row is missing + * @param exchange amount calculated by exchange + * @param auditor amount calculated by auditor + * @param profitable 1 if @a exchange being larger than @a auditor is + * profitable for the exchange for this operation, + * -1 if @a exchange being smaller than @a auditor is + * profitable for the exchange, and 0 if it is unclear + */ +static void +report_amount_arithmetic_inconsistency (const char *operation, + uint64_t rowid, + const struct + TALER_Amount *exchange, + const struct + TALER_Amount *auditor, + int profitable) +{ + struct TALER_Amount delta; + struct TALER_Amount *target; + + if (0 < TALER_amount_cmp (exchange, + auditor)) + { + /* exchange > auditor */ + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + exchange, + auditor)); + } + else + { + /* auditor < exchange */ + profitable = -profitable; + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + auditor, + exchange)); + } + TALER_ARL_report (report_amount_arithmetic_inconsistencies, + json_pack ("{s:s, s:I, s:o, s:o, s:I}", + "operation", operation, + "rowid", (json_int_t) rowid, + "exchange", TALER_JSON_from_amount (exchange), + "auditor", TALER_JSON_from_amount (auditor), + "profitable", (json_int_t) profitable)); + if (0 != profitable) + { + target = (1 == profitable) + ? &total_arithmetic_delta_plus + : &total_arithmetic_delta_minus; + GNUNET_break (GNUNET_OK == + TALER_amount_add (target, + target, + &delta)); + } +} + + +/** + * Report a (serious) inconsistency in the exchange's database. + * + * @param table affected table + * @param rowid affected row, UINT64_MAX if row is missing + * @param diagnostic message explaining the problem + */ +static void +report_row_inconsistency (const char *table, + uint64_t rowid, + const char *diagnostic) +{ + TALER_ARL_report (report_row_inconsistencies, + json_pack ("{s:s, s:I, s:s}", + "table", table, + "row", (json_int_t) rowid, + "diagnostic", diagnostic)); +} + + +/* ***************************** Analyze reserves ************************ */ +/* This logic checks the reserves_in, reserves_out and reserves-tables */ + +/** + * Summary data we keep per reserve. + */ +struct ReserveSummary +{ + /** + * Public key of the reserve. + * Always set when the struct is first initialized. + */ + struct TALER_ReservePublicKeyP reserve_pub; + + /** + * Sum of all incoming transfers during this transaction. + * Updated only in #handle_reserve_in(). + */ + struct TALER_Amount total_in; + + /** + * Sum of all outgoing transfers during this transaction (includes fees). + * Updated only in #handle_reserve_out(). + */ + struct TALER_Amount total_out; + + /** + * Sum of withdraw fees encountered during this transaction. + */ + struct TALER_Amount total_fee; + + /** + * Previous balance of the reserve as remembered by the auditor. + * (updated based on @e total_in and @e total_out at the end). + */ + struct TALER_Amount a_balance; + + /** + * Previous withdraw fee balance of the reserve, as remembered by the auditor. + * (updated based on @e total_fee at the end). + */ + struct TALER_Amount a_withdraw_fee_balance; + + /** + * Previous reserve expiration data, as remembered by the auditor. + * (updated on-the-fly in #handle_reserve_in()). + */ + struct GNUNET_TIME_Absolute a_expiration_date; + + /** + * Which account did originally put money into the reserve? + */ + char *sender_account; + + /** + * Did we have a previous reserve info? Used to decide between + * UPDATE and INSERT later. Initialized in + * #load_auditor_reserve_summary() together with the a-* values + * (if available). + */ + int had_ri; + +}; + + +/** + * Load the auditor's remembered state about the reserve into @a rs. + * The "total_in" and "total_out" amounts of @a rs must already be + * initialized (so we can determine the currency). + * + * @param[in,out] rs reserve summary to (fully) initialize + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +load_auditor_reserve_summary (struct ReserveSummary *rs) +{ + enum GNUNET_DB_QueryStatus qs; + uint64_t rowid; + + qs = TALER_ARL_adb->get_reserve_info (TALER_ARL_adb->cls, + TALER_ARL_asession, + &rs->reserve_pub, + &TALER_ARL_master_pub, + &rowid, + &rs->a_balance, + &rs->a_withdraw_fee_balance, + &rs->a_expiration_date, + &rs->sender_account); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + rs->had_ri = GNUNET_NO; + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (rs->total_in.currency, + &rs->a_balance)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (rs->total_in.currency, + &rs->a_withdraw_fee_balance)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Creating fresh reserve `%s' with starting balance %s\n", + TALER_B2S (&rs->reserve_pub), + TALER_amount2s (&rs->a_balance)); + return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; + } + rs->had_ri = GNUNET_YES; + if ( (GNUNET_YES != + TALER_amount_cmp_currency (&rs->a_balance, + &rs->a_withdraw_fee_balance)) || + (GNUNET_YES != + TALER_amount_cmp_currency (&rs->total_in, + &rs->a_balance)) ) + { + GNUNET_break (0); + return GNUNET_DB_STATUS_HARD_ERROR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Auditor remembers reserve `%s' has balance %s\n", + TALER_B2S (&rs->reserve_pub), + TALER_amount2s (&rs->a_balance)); + return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; +} + + +/** + * Closure to the various callbacks we make while checking a reserve. + */ +struct ReserveContext +{ + /** + * Map from hash of reserve's public key to a `struct ReserveSummary`. + */ + struct GNUNET_CONTAINER_MultiHashMap *reserves; + + /** + * Map from hash of denomination's public key to a + * static string "revoked" for keys that have been revoked, + * or "master signature invalid" in case the revocation is + * there but bogus. + */ + struct GNUNET_CONTAINER_MultiHashMap *revoked; + + /** + * Transaction status code, set to error codes if applicable. + */ + enum GNUNET_DB_QueryStatus qs; + +}; + + +/** + * Function called with details about incoming wire transfers. + * + * @param cls our `struct ReserveContext` + * @param rowid unique serial ID for the refresh session in our DB + * @param reserve_pub public key of the reserve (also the WTID) + * @param credit amount that was received + * @param sender_account_details information about the sender's bank account + * @param wire_reference unique reference identifying the wire transfer + * @param execution_date when did we receive the funds + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +handle_reserve_in (void *cls, + uint64_t rowid, + const struct TALER_ReservePublicKeyP *reserve_pub, + const struct TALER_Amount *credit, + const char *sender_account_details, + uint64_t wire_reference, + struct GNUNET_TIME_Absolute execution_date) +{ + struct ReserveContext *rc = cls; + struct GNUNET_HashCode key; + struct ReserveSummary *rs; + struct GNUNET_TIME_Absolute expiry; + enum GNUNET_DB_QueryStatus qs; + + (void) wire_reference; + /* should be monotonically increasing */ + GNUNET_assert (rowid >= ppr.last_reserve_in_serial_id); + ppr.last_reserve_in_serial_id = rowid + 1; + + GNUNET_CRYPTO_hash (reserve_pub, + sizeof (*reserve_pub), + &key); + rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, + &key); + if (NULL == rs) + { + rs = GNUNET_new (struct ReserveSummary); + rs->sender_account = GNUNET_strdup (sender_account_details); + rs->reserve_pub = *reserve_pub; + rs->total_in = *credit; + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (credit->currency, + &rs->total_out)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (credit->currency, + &rs->total_fee)); + if (0 > (qs = load_auditor_reserve_summary (rs))) + { + GNUNET_break (0); + GNUNET_free (rs); + rc->qs = qs; + return GNUNET_SYSERR; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (rc->reserves, + &key, + rs, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + else + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&rs->total_in, + &rs->total_in, + credit)); + if (NULL == rs->sender_account) + rs->sender_account = GNUNET_strdup (sender_account_details); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Additional incoming wire transfer for reserve `%s' of %s\n", + TALER_B2S (reserve_pub), + TALER_amount2s (credit)); + expiry = GNUNET_TIME_absolute_add (execution_date, + idle_reserve_expiration_time); + rs->a_expiration_date = GNUNET_TIME_absolute_max (rs->a_expiration_date, + expiry); + return GNUNET_OK; +} + + +/** + * Function called with details about withdraw operations. Verifies + * the signature and updates the reserve's balance. + * + * @param cls our `struct ReserveContext` + * @param rowid unique serial ID for the refresh session in our DB + * @param h_blind_ev blinded hash of the coin's public key + * @param denom_pub public denomination key of the deposited coin + * @param reserve_pub public key of the reserve + * @param reserve_sig signature over the withdraw operation + * @param execution_date when did the wallet withdraw the coin + * @param amount_with_fee amount that was withdrawn + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +handle_reserve_out (void *cls, + uint64_t rowid, + const struct GNUNET_HashCode *h_blind_ev, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_ReservePublicKeyP *reserve_pub, + const struct TALER_ReserveSignatureP *reserve_sig, + struct GNUNET_TIME_Absolute execution_date, + const struct TALER_Amount *amount_with_fee) +{ + struct ReserveContext *rc = cls; + struct TALER_WithdrawRequestPS wsrd; + struct GNUNET_HashCode key; + struct ReserveSummary *rs; + const struct TALER_DenominationKeyValidityPS *issue; + struct TALER_Amount withdraw_fee; + struct GNUNET_TIME_Absolute valid_start; + struct GNUNET_TIME_Absolute expire_withdraw; + enum GNUNET_DB_QueryStatus qs; + + /* should be monotonically increasing */ + GNUNET_assert (rowid >= ppr.last_reserve_out_serial_id); + ppr.last_reserve_out_serial_id = rowid + 1; + + /* lookup denomination pub data (make sure denom_pub is valid, establish fees) */ + qs = TALER_ARL_get_denomination_info (denom_pub, + &issue, + &wsrd.h_denomination_pub); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + if (GNUNET_DB_STATUS_HARD_ERROR == qs) + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Hard database error trying to get denomination %s (%s) from database!\n", + TALER_B2S (denom_pub), + TALER_amount2s (amount_with_fee)); + rc->qs = qs; + return GNUNET_SYSERR; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("withdraw", + rowid, + "denomination key not found"); + return GNUNET_OK; + } + + /* check that execution date is within withdraw range for denom_pub */ + valid_start = GNUNET_TIME_absolute_ntoh (issue->start); + expire_withdraw = GNUNET_TIME_absolute_ntoh (issue->expire_withdraw); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Checking withdraw timing: %llu, expire: %llu, timing: %llu\n", + (unsigned long long) valid_start.abs_value_us, + (unsigned long long) expire_withdraw.abs_value_us, + (unsigned long long) execution_date.abs_value_us); + if ( (valid_start.abs_value_us > execution_date.abs_value_us) || + (expire_withdraw.abs_value_us < execution_date.abs_value_us) ) + { + TALER_ARL_report (denomination_key_validity_withdraw_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o}", + "row", (json_int_t) rowid, + "execution_date", + TALER_ARL_json_from_time_abs (execution_date), + "reserve_pub", GNUNET_JSON_from_data_auto ( + reserve_pub), + "denompub_h", GNUNET_JSON_from_data_auto ( + &wsrd.h_denomination_pub))); + } + + /* check reserve_sig */ + wsrd.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_RESERVE_WITHDRAW); + wsrd.purpose.size = htonl (sizeof (wsrd)); + wsrd.reserve_pub = *reserve_pub; + TALER_amount_hton (&wsrd.amount_with_fee, + amount_with_fee); + wsrd.withdraw_fee = issue->fee_withdraw; + wsrd.h_coin_envelope = *h_blind_ev; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_RESERVE_WITHDRAW, + &wsrd.purpose, + &reserve_sig->eddsa_signature, + &reserve_pub->eddsa_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "withdraw", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount ( + amount_with_fee), + "key_pub", GNUNET_JSON_from_data_auto ( + reserve_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount_with_fee)); + return GNUNET_OK; + } + + GNUNET_CRYPTO_hash (reserve_pub, + sizeof (*reserve_pub), + &key); + rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, + &key); + if (NULL == rs) + { + rs = GNUNET_new (struct ReserveSummary); + rs->reserve_pub = *reserve_pub; + rs->total_out = *amount_with_fee; + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount_with_fee->currency, + &rs->total_in)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount_with_fee->currency, + &rs->total_fee)); + qs = load_auditor_reserve_summary (rs); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + GNUNET_free (rs); + rc->qs = qs; + return GNUNET_SYSERR; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (rc->reserves, + &key, + rs, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + else + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&rs->total_out, + &rs->total_out, + amount_with_fee)); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Reserve `%s' reduced by %s from withdraw\n", + TALER_B2S (reserve_pub), + TALER_amount2s (amount_with_fee)); + TALER_amount_ntoh (&withdraw_fee, + &issue->fee_withdraw); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Increasing withdraw profits by fee %s\n", + TALER_amount2s (&withdraw_fee)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&rs->total_fee, + &rs->total_fee, + &withdraw_fee)); + + return GNUNET_OK; +} + + +/** + * Function called with details about withdraw operations. Verifies + * the signature and updates the reserve's balance. + * + * @param cls our `struct ReserveContext` + * @param rowid unique serial ID for the refresh session in our DB + * @param timestamp when did we receive the recoup request + * @param amount how much should be added back to the reserve + * @param reserve_pub public key of the reserve + * @param coin public information about the coin, denomination signature is + * already verified in #check_recoup() + * @param denom_pub public key of the denomionation of @a coin + * @param coin_sig signature with @e coin_pub of type #TALER_SIGNATURE_WALLET_COIN_RECOUP + * @param coin_blind blinding factor used to blind the coin + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +handle_recoup_by_reserve (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute timestamp, + const struct TALER_Amount *amount, + const struct TALER_ReservePublicKeyP *reserve_pub, + const struct TALER_CoinPublicInfo *coin, + const struct TALER_DenominationPublicKey *denom_pub, + const struct TALER_CoinSpendSignatureP *coin_sig, + const struct + TALER_DenominationBlindingKeyP *coin_blind) +{ + struct ReserveContext *rc = cls; + struct GNUNET_HashCode key; + struct ReserveSummary *rs; + struct GNUNET_TIME_Absolute expiry; + struct TALER_RecoupRequestPS pr; + struct TALER_MasterSignatureP msig; + uint64_t rev_rowid; + enum GNUNET_DB_QueryStatus qs; + const char *rev; + + (void) denom_pub; + /* should be monotonically increasing */ + GNUNET_assert (rowid >= ppr.last_reserve_recoup_serial_id); + ppr.last_reserve_recoup_serial_id = rowid + 1; + /* We know that denom_pub matches denom_pub_hash because this + is how the SQL statement joined the tables. */ + pr.h_denom_pub = coin->denom_pub_hash; + pr.purpose.purpose = htonl (TALER_SIGNATURE_WALLET_COIN_RECOUP); + pr.purpose.size = htonl (sizeof (pr)); + pr.coin_pub = coin->coin_pub; + pr.coin_blind = *coin_blind; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify (TALER_SIGNATURE_WALLET_COIN_RECOUP, + &pr.purpose, + &coin_sig->eddsa_signature, + &coin->coin_pub.eddsa_pub)) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "recoup", + "row", (json_int_t) rowid, + "loss", TALER_JSON_from_amount (amount), + "key_pub", GNUNET_JSON_from_data_auto ( + &coin->coin_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount)); + } + + /* check that the coin was eligible for recoup!*/ + rev = GNUNET_CONTAINER_multihashmap_get (rc->revoked, + &pr.h_denom_pub); + if (NULL == rev) + { + qs = TALER_ARL_edb->get_denomination_revocation (TALER_ARL_edb->cls, + TALER_ARL_esession, + &pr.h_denom_pub, + &msig, + &rev_rowid); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + rc->qs = qs; + return GNUNET_SYSERR; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + report_row_inconsistency ("recoup", + rowid, + "denomination key not in revocation set"); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_irregular_recoups, + &total_irregular_recoups, + amount)); + } + else + { + /* verify msig */ + struct TALER_MasterDenominationKeyRevocationPS kr; + + kr.purpose.purpose = htonl ( + TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED); + kr.purpose.size = htonl (sizeof (kr)); + kr.h_denom_pub = pr.h_denom_pub; + if (GNUNET_OK != + GNUNET_CRYPTO_eddsa_verify ( + TALER_SIGNATURE_MASTER_DENOMINATION_KEY_REVOKED, + &kr.purpose, + &msig.eddsa_signature, + &TALER_ARL_master_pub.eddsa_pub)) + { + rev = "master signature invalid"; + } + else + { + rev = "revoked"; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (rc->revoked, + &pr.h_denom_pub, + (void *) rev, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + } + else + { + rev_rowid = 0; /* reported elsewhere */ + } + if ( (NULL != rev) && + (0 == strcmp (rev, "master signature invalid")) ) + { + TALER_ARL_report (report_bad_sig_losses, + json_pack ("{s:s, s:I, s:o, s:o}", + "operation", "recoup-master", + "row", (json_int_t) rev_rowid, + "loss", TALER_JSON_from_amount (amount), + "key_pub", GNUNET_JSON_from_data_auto ( + &TALER_ARL_master_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_sig_loss, + &total_bad_sig_loss, + amount)); + } + + GNUNET_CRYPTO_hash (reserve_pub, + sizeof (*reserve_pub), + &key); + rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, + &key); + if (NULL == rs) + { + rs = GNUNET_new (struct ReserveSummary); + rs->reserve_pub = *reserve_pub; + rs->total_in = *amount; + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount->currency, + &rs->total_out)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount->currency, + &rs->total_fee)); + qs = load_auditor_reserve_summary (rs); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + GNUNET_free (rs); + rc->qs = qs; + return GNUNET_SYSERR; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (rc->reserves, + &key, + rs, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + else + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&rs->total_in, + &rs->total_in, + amount)); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Additional /recoup value to for reserve `%s' of %s\n", + TALER_B2S (reserve_pub), + TALER_amount2s (amount)); + expiry = GNUNET_TIME_absolute_add (timestamp, + idle_reserve_expiration_time); + rs->a_expiration_date = GNUNET_TIME_absolute_max (rs->a_expiration_date, + expiry); + return GNUNET_OK; +} + + +/** + * Obtain the closing fee for a transfer at @a time for target + * @a receiver_account. + * + * @param receiver_account payto:// URI of the target account + * @param atime when was the transfer made + * @param[out] fee set to the closing fee + * @return #GNUNET_OK on success + */ +static int +get_closing_fee (const char *receiver_account, + struct GNUNET_TIME_Absolute atime, + struct TALER_Amount *fee) +{ + struct TALER_MasterSignatureP master_sig; + struct GNUNET_TIME_Absolute start_date; + struct GNUNET_TIME_Absolute end_date; + struct TALER_Amount wire_fee; + char *method; + + method = TALER_payto_get_method (receiver_account); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Method is `%s'\n", + method); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != + TALER_ARL_edb->get_wire_fee (TALER_ARL_edb->cls, + TALER_ARL_esession, + method, + atime, + &start_date, + &end_date, + &wire_fee, + fee, + &master_sig)) + { + report_row_inconsistency ("closing-fee", + atime.abs_value_us, + "closing fee unavailable at given time"); + GNUNET_free (method); + return GNUNET_SYSERR; + } + GNUNET_free (method); + return GNUNET_OK; +} + + +/** + * Function called about reserve closing operations + * the aggregator triggered. + * + * @param cls closure + * @param rowid row identifier used to uniquely identify the reserve closing operation + * @param execution_date when did we execute the close operation + * @param amount_with_fee how much did we debit the reserve + * @param closing_fee how much did we charge for closing the reserve + * @param reserve_pub public key of the reserve + * @param receiver_account where did we send the funds + * @param transfer_details details about the wire transfer + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +handle_reserve_closed (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute execution_date, + const struct TALER_Amount *amount_with_fee, + const struct TALER_Amount *closing_fee, + const struct TALER_ReservePublicKeyP *reserve_pub, + const char *receiver_account, + const struct + TALER_WireTransferIdentifierRawP *transfer_details) +{ + struct ReserveContext *rc = cls; + struct GNUNET_HashCode key; + struct ReserveSummary *rs; + enum GNUNET_DB_QueryStatus qs; + + (void) transfer_details; + /* should be monotonically increasing */ + GNUNET_assert (rowid >= ppr.last_reserve_close_serial_id); + ppr.last_reserve_close_serial_id = rowid + 1; + + GNUNET_CRYPTO_hash (reserve_pub, + sizeof (*reserve_pub), + &key); + rs = GNUNET_CONTAINER_multihashmap_get (rc->reserves, + &key); + if (NULL == rs) + { + rs = GNUNET_new (struct ReserveSummary); + rs->reserve_pub = *reserve_pub; + rs->total_out = *amount_with_fee; + rs->total_fee = *closing_fee; + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (amount_with_fee->currency, + &rs->total_in)); + qs = load_auditor_reserve_summary (rs); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + GNUNET_free (rs); + rc->qs = qs; + return GNUNET_SYSERR; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_put (rc->reserves, + &key, + rs, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)); + } + else + { + struct TALER_Amount expected_fee; + + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&rs->total_out, + &rs->total_out, + amount_with_fee)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&rs->total_fee, + &rs->total_fee, + closing_fee)); + /* verify closing_fee is correct! */ + if (GNUNET_OK != + get_closing_fee (receiver_account, + execution_date, + &expected_fee)) + { + GNUNET_break (0); + } + else if (0 != TALER_amount_cmp (&expected_fee, + closing_fee)) + { + report_amount_arithmetic_inconsistency ( + "closing aggregation fee", + rowid, + closing_fee, + &expected_fee, + 1); + } + } + if (NULL == rs->sender_account) + { + GNUNET_break (GNUNET_NO == rs->had_ri); + report_row_inconsistency ("reserves_close", + rowid, + "target account not verified, auditor does not know reserve"); + } + else if (0 != strcmp (rs->sender_account, + receiver_account)) + { + report_row_inconsistency ("reserves_close", + rowid, + "target account does not match origin account"); + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Additional closing operation for reserve `%s' of %s\n", + TALER_B2S (reserve_pub), + TALER_amount2s (amount_with_fee)); + return GNUNET_OK; +} + + +/** + * Check that the reserve summary matches what the exchange database + * thinks about the reserve, and update our own state of the reserve. + * + * Remove all reserves that we are happy with from the DB. + * + * @param cls our `struct ReserveContext` + * @param key hash of the reserve public key + * @param value a `struct ReserveSummary` + * @return #GNUNET_OK to process more entries + */ +static int +verify_reserve_balance (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct ReserveContext *rc = cls; + struct ReserveSummary *rs = value; + struct TALER_EXCHANGEDB_Reserve reserve; + struct TALER_Amount balance; + struct TALER_Amount nbalance; + struct TALER_Amount cfee; + enum GNUNET_DB_QueryStatus qs; + int ret; + + ret = GNUNET_OK; + reserve.pub = rs->reserve_pub; + qs = TALER_ARL_edb->reserves_get (TALER_ARL_edb->cls, + TALER_ARL_esession, + &reserve); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != qs) + { + char *diag; + + GNUNET_asprintf (&diag, + "Failed to find summary for reserve `%s'\n", + TALER_B2S (&rs->reserve_pub)); + report_row_inconsistency ("reserve-summary", + UINT64_MAX, + diag); + GNUNET_free (diag); + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) + { + GNUNET_break (0); + qs = GNUNET_DB_STATUS_HARD_ERROR; + } + rc->qs = qs; + return GNUNET_OK; + } + + if (GNUNET_OK != + TALER_amount_add (&balance, + &rs->total_in, + &rs->a_balance)) + { + GNUNET_break (0); + goto cleanup; + } + + if (GNUNET_SYSERR == + TALER_amount_subtract (&nbalance, + &balance, + &rs->total_out)) + { + struct TALER_Amount loss; + + GNUNET_break (GNUNET_SYSERR != + TALER_amount_subtract (&loss, + &rs->total_out, + &balance)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_balance_insufficient_loss, + &total_balance_insufficient_loss, + &loss)); + TALER_ARL_report ( + report_reserve_balance_insufficient_inconsistencies, + json_pack ("{s:o, s:o}", + "reserve_pub", + GNUNET_JSON_from_data_auto (&rs->reserve_pub), + "loss", + TALER_JSON_from_amount (&loss))); + goto cleanup; + } + if (0 != TALER_amount_cmp (&nbalance, + &reserve.balance)) + { + struct TALER_Amount delta; + + if (0 < TALER_amount_cmp (&nbalance, + &reserve.balance)) + { + /* balance > reserve.balance */ + GNUNET_assert (GNUNET_OK == + TALER_amount_subtract (&delta, + &nbalance, + &reserve.balance)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&total_balance_summary_delta_plus, + &total_balance_summary_delta_plus, + &delta)); + } + else + { + /* balance < reserve.balance */ + GNUNET_assert (GNUNET_OK == + TALER_amount_subtract (&delta, + &reserve.balance, + &nbalance)); + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&total_balance_summary_delta_minus, + &total_balance_summary_delta_minus, + &delta)); + } + TALER_ARL_report ( + report_reserve_balance_summary_wrong_inconsistencies, + json_pack ("{s:o, s:o, s:o}", + "reserve_pub", + GNUNET_JSON_from_data_auto (&rs->reserve_pub), + "exchange", + TALER_JSON_from_amount (&reserve.balance), + "auditor", + TALER_JSON_from_amount (&nbalance))); + goto cleanup; + } + + /* Check that reserve is being closed if it is past its expiration date */ + + if (CLOSING_GRACE_PERIOD.rel_value_us < + GNUNET_TIME_absolute_get_duration (rs->a_expiration_date).rel_value_us) + { + if ( (NULL != rs->sender_account) && + (GNUNET_OK == + get_closing_fee (rs->sender_account, + rs->a_expiration_date, + &cfee)) ) + { + if (1 == TALER_amount_cmp (&nbalance, + &cfee)) + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&total_balance_reserve_not_closed, + &total_balance_reserve_not_closed, + &nbalance)); + TALER_ARL_report (report_reserve_not_closed_inconsistencies, + json_pack ("{s:o, s:o, s:o}", + "reserve_pub", + GNUNET_JSON_from_data_auto ( + &rs->reserve_pub), + "balance", + TALER_JSON_from_amount (&nbalance), + "expiration_time", + TALER_ARL_json_from_time_abs ( + rs->a_expiration_date))); + } + } + else + { + GNUNET_assert (GNUNET_OK == + TALER_amount_add (&total_balance_reserve_not_closed, + &total_balance_reserve_not_closed, + &nbalance)); + TALER_ARL_report (report_reserve_not_closed_inconsistencies, + json_pack ("{s:o, s:o, s:o, s:s}", + "reserve_pub", + GNUNET_JSON_from_data_auto ( + &rs->reserve_pub), + "balance", + TALER_JSON_from_amount (&nbalance), + "expiration_time", + TALER_ARL_json_from_time_abs ( + rs->a_expiration_date), + "diagnostic", + "could not determine closing fee")); + } + } + + /* Add withdraw fees we encountered to totals */ + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Reserve reserve `%s' made %s in withdraw fees\n", + TALER_B2S (&rs->reserve_pub), + TALER_amount2s (&rs->total_fee)); + if (GNUNET_YES != + TALER_amount_add (&rs->a_withdraw_fee_balance, + &rs->a_withdraw_fee_balance, + &rs->total_fee)) + { + GNUNET_break (0); + ret = GNUNET_SYSERR; + goto cleanup; + } + if ( (GNUNET_YES != + TALER_amount_add (&total_escrow_balance, + &total_escrow_balance, + &rs->total_in)) || + (GNUNET_SYSERR == + TALER_amount_subtract (&total_escrow_balance, + &total_escrow_balance, + &rs->total_out)) || + (GNUNET_YES != + TALER_amount_add (&total_withdraw_fee_income, + &total_withdraw_fee_income, + &rs->total_fee)) ) + { + GNUNET_break (0); + ret = GNUNET_SYSERR; + goto cleanup; + } + + if ( (0ULL == balance.value) && + (0U == balance.fraction) ) + { + /* balance is zero, drop reserve details (and then do not update/insert) */ + if (rs->had_ri) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Final balance of reserve `%s' is %s, dropping it\n", + TALER_B2S (&rs->reserve_pub), + TALER_amount2s (&nbalance)); + qs = TALER_ARL_adb->del_reserve_info (TALER_ARL_adb->cls, + TALER_ARL_asession, + &rs->reserve_pub, + &TALER_ARL_master_pub); + if (0 >= qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + ret = GNUNET_SYSERR; + rc->qs = qs; + goto cleanup; + } + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Final balance of reserve `%s' is %s, no need to remember it\n", + TALER_B2S (&rs->reserve_pub), + TALER_amount2s (&nbalance)); + } + ret = GNUNET_OK; + goto cleanup; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Remembering final balance of reserve `%s' as %s\n", + TALER_B2S (&rs->reserve_pub), + TALER_amount2s (&nbalance)); + + if (rs->had_ri) + qs = TALER_ARL_adb->update_reserve_info (TALER_ARL_adb->cls, + TALER_ARL_asession, + &rs->reserve_pub, + &TALER_ARL_master_pub, + &nbalance, + &rs->a_withdraw_fee_balance, + rs->a_expiration_date); + else + qs = TALER_ARL_adb->insert_reserve_info (TALER_ARL_adb->cls, + TALER_ARL_asession, + &rs->reserve_pub, + &TALER_ARL_master_pub, + &nbalance, + &rs->a_withdraw_fee_balance, + rs->a_expiration_date, + rs->sender_account); + if (0 >= qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + ret = GNUNET_SYSERR; + rc->qs = qs; + } +cleanup: + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (rc->reserves, + key, + rs)); + GNUNET_free_non_null (rs->sender_account); + GNUNET_free (rs); + return ret; +} + + +/** + * Analyze reserves for being well-formed. + * + * @param cls NULL + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +analyze_reserves (void *cls) +{ + struct ReserveContext rc; + enum GNUNET_DB_QueryStatus qsx; + enum GNUNET_DB_QueryStatus qs; + enum GNUNET_DB_QueryStatus qsp; + + (void) cls; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Analyzing reserves\n"); + qsp = TALER_ARL_adb->get_auditor_progress_reserve (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppr); + if (0 > qsp) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsp); + return qsp; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsp) + { + GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, + _ ( + "First analysis using this auditor, starting audit from scratch\n")); + } + else + { + ppr_start = ppr; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Resuming reserve audit at %llu/%llu/%llu/%llu\n"), + (unsigned long long) ppr.last_reserve_in_serial_id, + (unsigned long long) ppr.last_reserve_out_serial_id, + (unsigned long long) ppr.last_reserve_recoup_serial_id, + (unsigned long long) ppr.last_reserve_close_serial_id); + } + rc.qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; + qsx = TALER_ARL_adb->get_reserve_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_escrow_balance, + &total_withdraw_fee_income); + if (qsx < 0) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx); + return qsx; + } + rc.reserves = GNUNET_CONTAINER_multihashmap_create (512, + GNUNET_NO); + rc.revoked = GNUNET_CONTAINER_multihashmap_create (4, + GNUNET_NO); + + qs = TALER_ARL_edb->select_reserves_in_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppr. + last_reserve_in_serial_id, + &handle_reserve_in, + &rc); + if (qs < 0) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + qs = TALER_ARL_edb->select_withdrawals_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppr. + last_reserve_out_serial_id, + &handle_reserve_out, + &rc); + if (qs < 0) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + qs = TALER_ARL_edb->select_recoup_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppr. + last_reserve_recoup_serial_id, + &handle_recoup_by_reserve, + &rc); + if (qs < 0) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + qs = TALER_ARL_edb->select_reserve_closed_above_serial_id (TALER_ARL_edb->cls, + TALER_ARL_esession, + ppr. + last_reserve_close_serial_id, + & + handle_reserve_closed, + &rc); + if (qs < 0) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + + GNUNET_CONTAINER_multihashmap_iterate (rc.reserves, + &verify_reserve_balance, + &rc); + GNUNET_break (0 == + GNUNET_CONTAINER_multihashmap_size (rc.reserves)); + GNUNET_CONTAINER_multihashmap_destroy (rc.reserves); + GNUNET_CONTAINER_multihashmap_destroy (rc.revoked); + + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT != rc.qs) + return qs; + + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsx) + { + qs = TALER_ARL_adb->insert_reserve_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_escrow_balance, + &total_withdraw_fee_income); + } + else + { + qs = TALER_ARL_adb->update_reserve_summary (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &total_escrow_balance, + &total_withdraw_fee_income); + } + if (0 >= qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsp) + qs = TALER_ARL_adb->update_auditor_progress_reserve (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppr); + else + qs = TALER_ARL_adb->insert_auditor_progress_reserve (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &ppr); + if (0 >= qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to update auditor DB, not recording progress\n"); + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _ ("Concluded reserve audit step at %llu/%llu/%llu/%llu\n"), + (unsigned long long) ppr.last_reserve_in_serial_id, + (unsigned long long) ppr.last_reserve_out_serial_id, + (unsigned long long) ppr.last_reserve_recoup_serial_id, + (unsigned long long) ppr.last_reserve_close_serial_id); + return GNUNET_DB_STATUS_SUCCESS_ONE_RESULT; +} + + +/** + * Main function that will be run. + * + * @param cls closure + * @param args remaining command-line arguments + * @param TALER_ARL_cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *TALER_ARL_cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + (void) cls; + (void) args; + (void) TALER_ARL_cfgfile; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Launching auditor\n"); + if (GNUNET_OK != + TALER_ARL_init (TALER_ARL_cfg)) + { + global_ret = 1; + return; + } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_time (TALER_ARL_cfg, + "exchangTALER_ARL_edb", + "IDLE_RESERVE_EXPIRATION_TIME", + &idle_reserve_expiration_time)) + { + GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, + "exchangTALER_ARL_edb", + "IDLE_RESERVE_EXPIRATION_TIME"); + global_ret = 1; + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting audit\n"); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_escrow_balance)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_irregular_recoups)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_withdraw_fee_income)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_balance_insufficient_loss)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_balance_summary_delta_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_balance_summary_delta_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_arithmetic_delta_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_arithmetic_delta_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_balance_reserve_not_closed)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_sig_loss)); + GNUNET_assert (NULL != + (report_row_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (denomination_key_validity_withdraw_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_reserve_balance_summary_wrong_inconsistencies + = + json_array ())); + GNUNET_assert (NULL != + (report_reserve_balance_insufficient_inconsistencies + = + json_array ())); + GNUNET_assert (NULL != + (report_reserve_not_closed_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_amount_arithmetic_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_bad_sig_losses = json_array ())); + if (GNUNET_OK != + TALER_ARL_setup_sessions_and_run (&analyze_reserves, + NULL)) + { + global_ret = 1; + return; + } + { + json_t *report; + + report = json_pack ("{s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:I," + " s:I, s:I, s:I, s:I, s:I," + " s:I, s:I }", + /* blocks #1 */ + "reserve_balance_insufficient_inconsistencies", + report_reserve_balance_insufficient_inconsistencies, + /* Tested in test-auditor.sh #3 */ + "total_loss_balance_insufficient", + TALER_JSON_from_amount ( + &total_balance_insufficient_loss), + /* Tested in test-auditor.sh #3 */ + "reserve_balance_summary_wrong_inconsistencies", + report_reserve_balance_summary_wrong_inconsistencies, + "total_balance_summary_delta_plus", + TALER_JSON_from_amount ( + &total_balance_summary_delta_plus), + "total_balance_summary_delta_minus", + TALER_JSON_from_amount ( + &total_balance_summary_delta_minus), + /* blocks #2 */ + "total_escrow_balance", + TALER_JSON_from_amount (&total_escrow_balance), + "total_withdraw_fee_income", + TALER_JSON_from_amount ( + &total_withdraw_fee_income), + /* Tested in test-auditor.sh #21 */ + "reserve_not_closed_inconsistencies", + report_reserve_not_closed_inconsistencies, + /* Tested in test-auditor.sh #21 */ + "total_balance_reserve_not_closed", + TALER_JSON_from_amount ( + &total_balance_reserve_not_closed), + /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ + "bad_sig_losses", + report_bad_sig_losses, + /* blocks #3 */ + /* Tested in test-auditor.sh #4/#5/#6/#7/#13 */ + "total_bad_sig_loss", + TALER_JSON_from_amount (&total_bad_sig_loss), + /* Tested in test-auditor.sh #14/#15 */ + "row_inconsistencies", + report_row_inconsistencies, + /* Tested in test-auditor.sh #23 */ + "denomination_key_validity_withdraw_inconsistencies", + denomination_key_validity_withdraw_inconsistencies, + "amount_arithmetic_inconsistencies", + report_amount_arithmetic_inconsistencies, + "total_arithmetic_delta_plus", + TALER_JSON_from_amount ( + &total_arithmetic_delta_plus), + /* blocks #4 */ + "total_arithmetic_delta_minus", + TALER_JSON_from_amount ( + &total_arithmetic_delta_minus), + "auditor_start_time", + TALER_ARL_json_from_time_abs ( + start_time), + "auditor_end_time", + TALER_ARL_json_from_time_abs ( + GNUNET_TIME_absolute_get ()), + "total_irregular_recoups", + TALER_JSON_from_amount ( + &total_irregular_recoups), + "start_ppr_reserve_in_serial_id", + (json_int_t) ppr_start.last_reserve_in_serial_id, + /* blocks #5 */ + "start_ppr_reserve_out_serial_id", + (json_int_t) ppr_start. + last_reserve_out_serial_id, + "start_ppr_reserve_recoup_serial_id", + (json_int_t) ppr_start. + last_reserve_recoup_serial_id, + "start_ppr_reserve_close_serial_id", + (json_int_t) ppr_start. + last_reserve_close_serial_id, + "end_ppr_reserve_in_serial_id", + (json_int_t) ppr.last_reserve_in_serial_id, + "end_ppr_reserve_out_serial_id", + (json_int_t) ppr.last_reserve_out_serial_id, + /* blocks #6 */ + "end_ppr_reserve_recoup_serial_id", + (json_int_t) ppr.last_reserve_recoup_serial_id, + "end_ppr_reserve_close_serial_id", + (json_int_t) ppr.last_reserve_close_serial_id + ); + GNUNET_break (NULL != report); + TALER_ARL_done (report); + } +} + + +/** + * The main function of the database initialization tool. + * Used to initialize the Taler Exchange's database. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + const struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_base32_auto ('m', + "exchange-key", + "KEY", + "public key of the exchange (Crockford base32 encoded)", + &TALER_ARL_master_pub), + GNUNET_GETOPT_option_flag ('r', + "TALER_ARL_restart", + "TALER_ARL_restart audit from the beginning (required on first run)", + &TALER_ARL_restart), + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_OPTION_END + }; + + /* force linker to link against libtalerutil; if we do + not do this, the linker may "optimize" libtalerutil + away and skip #TALER_OS_init(), which we do need */ + (void) TALER_project_data_default (); + GNUNET_assert (GNUNET_OK == + GNUNET_log_setup ("taler-auditor-reserves", + "MESSAGE", + NULL)); + if (GNUNET_OK != + GNUNET_PROGRAM_run (argc, + argv, + "taler-auditor-reserves", + "Audit Taler exchange reserve handling", + options, + &run, + NULL)) + return 1; + return global_ret; +} + + +/* end of taler-auditor-reserves.c */ diff --git a/src/auditor/taler-helper-auditor-wire.c b/src/auditor/taler-helper-auditor-wire.c new file mode 100644 index 000000000..bc982655e --- /dev/null +++ b/src/auditor/taler-helper-auditor-wire.c @@ -0,0 +1,2229 @@ +/* + This file is part of TALER + Copyright (C) 2017-2020 Taler Systems SA + + TALER is free software; you can redistribute it and/or modify it under the + terms of the GNU General Public License as published by the Free Software + Foundation; either version 3, or (at your option) any later version. + + TALER is distributed in the hope that it will be useful, but WITHOUT ANY + WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR + A PARTICULAR PURPOSE. See the GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along with + TALER; see the file COPYING. If not, see +*/ +/** + * @file auditor/taler-helper-auditor-wire.c + * @brief audits that wire transfers match those from an exchange database. + * @author Christian Grothoff + * + * - First, this auditor verifies that 'reserves_in' actually matches + * the incoming wire transfers from the bank. + * - Second, we check that the outgoing wire transfers match those + * given in the 'wire_out' and 'reserve_closures' tables + * - Finally, we check that all wire transfers that should have been made, + * were actually made + */ +#include "platform.h" +#include +#include +#include "taler_auditordb_plugin.h" +#include "taler_exchangedb_lib.h" +#include "taler_json_lib.h" +#include "taler_bank_service.h" +#include "taler_signatures.h" +#include "report-lib.h" + + +/** + * How much time do we allow the aggregator to lag behind? If + * wire transfers should have been made more than #GRACE_PERIOD + * before, we issue warnings. + */ +#define GRACE_PERIOD GNUNET_TIME_UNIT_HOURS + +/** + * How much do we allow the bank and the exchange to disagree about + * timestamps? Should be sufficiently large to avoid bogus reports from deltas + * created by imperfect clock synchronization and network delay. + */ +#define TIME_TOLERANCE GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, \ + 15) + + +/** + * Information we keep for each supported account. + */ +struct WireAccount +{ + /** + * Accounts are kept in a DLL. + */ + struct WireAccount *next; + + /** + * Plugins are kept in a DLL. + */ + struct WireAccount *prev; + + /** + * Authentication data for the account. + */ + struct TALER_BANK_AuthenticationData auth; + + /** + * Name of the section that configures this account. + */ + char *section_name; + + /** + * Active wire request for the transaction history. + */ + struct TALER_BANK_CreditHistoryHandle *chh; + + /** + * Active wire request for the transaction history. + */ + struct TALER_BANK_DebitHistoryHandle *dhh; + + /** + * Progress point for this account. + */ + struct TALER_AUDITORDB_WireAccountProgressPoint pp; + + /** + * Initial progress point for this account. + */ + struct TALER_AUDITORDB_WireAccountProgressPoint start_pp; + + /** + * Where we are in the inbound (CREDIT) transaction history. + */ + uint64_t in_wire_off; + + /** + * Where we are in the inbound (DEBIT) transaction history. + */ + uint64_t out_wire_off; + + /** + * We should check for inbound transactions to this account. + */ + int watch_credit; + + /** + * We should check for outbound transactions from this account. + */ + int watch_debit; + + /** + * Return value when we got this account's progress point. + */ + enum GNUNET_DB_QueryStatus qsx; +}; + + +/** + * Information we track for a reserve being closed. + */ +struct ReserveClosure +{ + /** + * Row in the reserves_closed table for this action. + */ + uint64_t rowid; + + /** + * When was the reserve closed? + */ + struct GNUNET_TIME_Absolute execution_date; + + /** + * Amount transferred (amount remaining minus fee). + */ + struct TALER_Amount amount; + + /** + * Target account where the money was sent. + */ + char *receiver_account; + + /** + * Wire transfer subject used. + */ + struct TALER_WireTransferIdentifierRawP wtid; +}; + + +/** + * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. + */ +static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; + +/** + * Return value from main(). + */ +static int global_ret; + +/** + * Map with information about incoming wire transfers. + * Maps hashes of the wire offsets to `struct ReserveInInfo`s. + */ +static struct GNUNET_CONTAINER_MultiHashMap *in_map; + +/** + * Map with information about outgoing wire transfers. + * Maps hashes of the wire subjects (in binary encoding) + * to `struct ReserveOutInfo`s. + */ +static struct GNUNET_CONTAINER_MultiHashMap *out_map; + +/** + * Head of list of wire accounts we still need to look at. + */ +static struct WireAccount *wa_head; + +/** + * Tail of list of wire accounts we still need to look at. + */ +static struct WireAccount *wa_tail; + +/** + * Query status for the incremental processing status in the auditordb. + * Return value from our call to the "get_wire_auditor_progress" function. + */ +static enum GNUNET_DB_QueryStatus qsx_gwap; + +/** + * Last reserve_in / wire_out serial IDs seen. + */ +static struct TALER_AUDITORDB_WireProgressPoint pp; + +/** + * Last reserve_in / wire_out serial IDs seen. + */ +static struct TALER_AUDITORDB_WireProgressPoint start_pp; + +/** + * Array of reports about row inconsitencies in wire_out table. + */ +static json_t *report_wire_out_inconsistencies; + +/** + * Array of reports about row inconsitencies in reserves_in table. + */ +static json_t *report_reserve_in_inconsistencies; + +/** + * Array of reports about wrong bank account being recorded for + * incoming wire transfers. + */ +static json_t *report_missattribution_in_inconsistencies; + +/** + * Array of reports about row inconcistencies. + */ +static json_t *report_row_inconsistencies; + +/** + * Array of reports about inconcistencies in the database about + * the incoming wire transfers (exchange is not exactly to blame). + */ +static json_t *report_wire_format_inconsistencies; + +/** + * Array of reports about minor row inconcistencies. + */ +static json_t *report_row_minor_inconsistencies; + +/** + * Array of reports about lagging transactions from deposits. + */ +static json_t *report_lags; + +/** + * Array of reports about lagging transactions from reserve closures. + */ +static json_t *report_closure_lags; + +/** + * Array of per-account progress data. + */ +static json_t *report_account_progress; + +/** + * Amount that is considered "tiny" + */ +static struct TALER_Amount tiny_amount; + +/** + * Total amount that was transferred too much from the exchange. + */ +static struct TALER_Amount total_bad_amount_out_plus; + +/** + * Total amount that was transferred too little from the exchange. + */ +static struct TALER_Amount total_bad_amount_out_minus; + +/** + * Total amount that was transferred too much to the exchange. + */ +static struct TALER_Amount total_bad_amount_in_plus; + +/** + * Total amount that was transferred too little to the exchange. + */ +static struct TALER_Amount total_bad_amount_in_minus; + +/** + * Total amount where the exchange has the wrong sender account + * for incoming funds and may thus wire funds to the wrong + * destination when closing the reserve. + */ +static struct TALER_Amount total_missattribution_in; + +/** + * Total amount which the exchange did not transfer in time. + */ +static struct TALER_Amount total_amount_lag; + +/** + * Total amount of reserve closures which the exchange did not transfer in time. + */ +static struct TALER_Amount total_closure_amount_lag; + +/** + * Total amount affected by wire format trouble.s + */ +static struct TALER_Amount total_wire_format_amount; + +/** + * Amount of zero in our TALER_ARL_currency. + */ +static struct TALER_Amount zero; + +/** + * Handle to the context for interacting with the bank. + */ +static struct GNUNET_CURL_Context *ctx; + +/** + * Scheduler context for running the @e ctx. + */ +static struct GNUNET_CURL_RescheduleContext *rc; + + +/* ***************************** Shutdown **************************** */ + +/** + * Entry in map with wire information we expect to obtain from the + * bank later. + */ +struct ReserveInInfo +{ + + /** + * Hash of expected row offset. + */ + struct GNUNET_HashCode row_off_hash; + + /** + * Expected details about the wire transfer. + * The member "account_url" is to be allocated + * at the end of this struct! + */ + struct TALER_BANK_CreditDetails details; + + /** + * RowID in reserves_in table. + */ + uint64_t rowid; + +}; + + +/** + * Entry in map with wire information we expect to obtain from the + * #TALER_ARL_edb later. + */ +struct ReserveOutInfo +{ + + /** + * Hash of the wire transfer subject. + */ + struct GNUNET_HashCode subject_hash; + + /** + * Expected details about the wire transfer. + */ + struct TALER_BANK_DebitDetails details; + +}; + + +/** + * Free entry in #in_map. + * + * @param cls NULL + * @param key unused key + * @param value the `struct ReserveInInfo` to free + * @return #GNUNET_OK + */ +static int +free_rii (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct ReserveInInfo *rii = value; + + (void) cls; + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (in_map, + key, + rii)); + GNUNET_free (rii); + return GNUNET_OK; +} + + +/** + * Free entry in #out_map. + * + * @param cls NULL + * @param key unused key + * @param value the `struct ReserveOutInfo` to free + * @return #GNUNET_OK + */ +static int +free_roi (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct ReserveOutInfo *roi = value; + + (void) cls; + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (out_map, + key, + roi)); + GNUNET_free (roi); + return GNUNET_OK; +} + + +/** + * Free entry in #reserve_closures. + * + * @param cls NULL + * @param key unused key + * @param value the `struct ReserveClosure` to free + * @return #GNUNET_OK + */ +static int +free_rc (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct ReserveClosure *rc = value; + + (void) cls; + GNUNET_assert (GNUNET_YES == + GNUNET_CONTAINER_multihashmap_remove (reserve_closures, + key, + rc)); + GNUNET_free (rc->receiver_account); + GNUNET_free (rc); + return GNUNET_OK; +} + + +/** + * Task run on shutdown. + * + * @param cls NULL + */ +static void +do_shutdown (void *cls) +{ + struct WireAccount *wa; + + (void) cls; + if (NULL != ctx) + { + GNUNET_CURL_fini (ctx); + ctx = NULL; + } + if (NULL != rc) + { + GNUNET_CURL_gnunet_rc_destroy (rc); + rc = NULL; + } + + if (NULL != report_row_inconsistencies) + { + json_t *report; + + GNUNET_assert (NULL != report_row_minor_inconsistencies); + report = json_pack ("{s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:o, s:o," + " s:o, s:o, s:o, s:I, s:I," + " s:o, s:o, s:o }", + /* blocks of 5 */ + /* Tested in test-auditor.sh #11, #15, #20 */ + "wire_out_amount_inconsistencies", + report_wire_out_inconsistencies, + "total_wire_out_delta_plus", + TALER_JSON_from_amount ( + &total_bad_amount_out_plus), + /* Tested in test-auditor.sh #11, #15, #19 */ + "total_wire_out_delta_minus", + TALER_JSON_from_amount ( + &total_bad_amount_out_minus), + /* Tested in test-auditor.sh #2 */ + "reserve_in_amount_inconsistencies", + report_reserve_in_inconsistencies, + /* Tested in test-auditor.sh #2 */ + "total_wire_in_delta_plus", + TALER_JSON_from_amount ( + &total_bad_amount_in_plus), + /* block */ + /* Tested in test-auditor.sh #3 */ + "total_wire_in_delta_minus", + TALER_JSON_from_amount ( + &total_bad_amount_in_minus), + /* Tested in test-auditor.sh #9 */ + "missattribution_in_inconsistencies", + report_missattribution_in_inconsistencies, + /* Tested in test-auditor.sh #9 */ + "total_missattribution_in", + TALER_JSON_from_amount ( + &total_missattribution_in), + "row_inconsistencies", + report_row_inconsistencies, + /* Tested in test-auditor.sh #10/#17 */ + "row_minor_inconsistencies", + report_row_minor_inconsistencies, + /* block */ + /* Tested in test-auditor.sh #19 */ + "total_wire_format_amount", + TALER_JSON_from_amount ( + &total_wire_format_amount), + /* Tested in test-auditor.sh #19 */ + "wire_format_inconsistencies", + report_wire_format_inconsistencies, + /* Tested in test-auditor.sh #1 */ + "total_amount_lag", + TALER_JSON_from_amount (&total_amount_lag), + /* Tested in test-auditor.sh #1 */ + "lag_details", + report_lags, + /* Tested in test-auditor.sh #22 */ + "total_closure_amount_lag", + TALER_JSON_from_amount ( + &total_closure_amount_lag), + /* blocks of 5 */ + /* Tested in test-auditor.sh #22 */ + "reserve_lag_details", + report_closure_lags, + "wire_auditor_start_time", + TALER_ARL_json_from_time_abs ( + start_time), + "wire_auditor_end_time", + TALER_ARL_json_from_time_abs ( + GNUNET_TIME_absolute_get ()), + "start_pp_reserve_close_uuid", + (json_int_t) start_pp.last_reserve_close_uuid, + "end_pp_reserve_close_uuid", + (json_int_t) pp.last_reserve_close_uuid, + /* blocks of 5 */ + "start_pp_last_timestamp", + TALER_ARL_json_from_time_abs ( + start_pp.last_timestamp), + "end_pp_last_timestamp", + TALER_ARL_json_from_time_abs ( + pp.last_timestamp), + "account_progress", + report_account_progress + ); + GNUNET_break (NULL != report); + TALER_ARL_done (report); + report_wire_out_inconsistencies = NULL; + report_reserve_in_inconsistencies = NULL; + report_row_inconsistencies = NULL; + report_row_minor_inconsistencies = NULL; + report_missattribution_in_inconsistencies = NULL; + report_lags = NULL; + report_closure_lags = NULL; + report_account_progress = NULL; + report_wire_format_inconsistencies = NULL; + } + else + { + TALER_ARL_done (NULL); + } + if (NULL != reserve_closures) + { + GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, + &free_rc, + NULL); + GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); + reserve_closures = NULL; + } + if (NULL != in_map) + { + GNUNET_CONTAINER_multihashmap_iterate (in_map, + &free_rii, + NULL); + GNUNET_CONTAINER_multihashmap_destroy (in_map); + in_map = NULL; + } + if (NULL != out_map) + { + GNUNET_CONTAINER_multihashmap_iterate (out_map, + &free_roi, + NULL); + GNUNET_CONTAINER_multihashmap_destroy (out_map); + out_map = NULL; + } + while (NULL != (wa = wa_head)) + { + if (NULL != wa->dhh) + { + TALER_BANK_debit_history_cancel (wa->dhh); + wa->dhh = NULL; + } + if (NULL != wa->chh) + { + TALER_BANK_credit_history_cancel (wa->chh); + wa->chh = NULL; + } + GNUNET_CONTAINER_DLL_remove (wa_head, + wa_tail, + wa); + TALER_BANK_auth_free (&wa->auth); + GNUNET_free (wa->section_name); + GNUNET_free (wa); + } +} + + +/* ***************************** Report logic **************************** */ + + +/** + * Detect any entries in #reserve_closures that were not yet + * observed on the wire transfer side and update the progress + * point accordingly. + * + * @param cls NULL + * @param key unused key + * @param value the `struct ReserveClosure` to free + * @return #GNUNET_OK + */ +static int +check_pending_rc (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct ReserveClosure *rc = value; + + (void) cls; + (void) key; + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_closure_amount_lag, + &total_closure_amount_lag, + &rc->amount)); + if ( (0 != rc->amount.value) || + (0 != rc->amount.fraction) ) + TALER_ARL_report (report_closure_lags, + json_pack ("{s:I, s:o, s:o, s:o, s:s}", + "row", (json_int_t) rc->rowid, + "amount", TALER_JSON_from_amount (&rc->amount), + "deadline", TALER_ARL_json_from_time_abs ( + rc->execution_date), + "wtid", GNUNET_JSON_from_data_auto (&rc->wtid), + "account", rc->receiver_account)); + pp.last_reserve_close_uuid + = GNUNET_MIN (pp.last_reserve_close_uuid, + rc->rowid); + return GNUNET_OK; +} + + +/** + * Compute the key under which a reserve closure for a given + * @a receiver_account and @a wtid would be stored. + * + * @param receiver_account payto://-URI of the account + * @param wtid wire transfer identifier used + * @param[out] key set to the key + */ +static void +hash_rc (const char *receiver_account, + const struct TALER_WireTransferIdentifierRawP *wtid, + struct GNUNET_HashCode *key) +{ + size_t slen = strlen (receiver_account); + char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; + + memcpy (buf, + wtid, + sizeof (*wtid)); + memcpy (&buf[sizeof (*wtid)], + receiver_account, + slen); + GNUNET_CRYPTO_hash (buf, + sizeof (buf), + key); +} + + +/* *************************** General transaction logic ****************** */ + +/** + * Commit the transaction, checkpointing our progress in the auditor + * DB. + * + * @param qs transaction status so far + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +commit (enum GNUNET_DB_QueryStatus qs) +{ + if (0 > qs) + { + if (GNUNET_DB_STATUS_SOFT_ERROR == qs) + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Serialization issue, not recording progress\n"); + else + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Hard error, not recording progress\n"); + TALER_ARL_adb->rollback (TALER_ARL_adb->cls, + TALER_ARL_asession); + TALER_ARL_edb->rollback (TALER_ARL_edb->cls, + TALER_ARL_esession); + return qs; + } + for (struct WireAccount *wa = wa_head; + NULL != wa; + wa = wa->next) + { + GNUNET_assert (0 == + json_array_append_new (report_account_progress, + json_pack ( + "{s:s, s:I, s:I, s:I, s:I}", + "account", + wa->section_name, + "start_reserve_in", + (json_int_t) wa->start_pp. + last_reserve_in_serial_id, + "end_reserve_in", + (json_int_t) wa->pp. + last_reserve_in_serial_id, + "start_wire_out", + (json_int_t) wa->start_pp. + last_wire_out_serial_id, + "end_wire_out", + (json_int_t) wa->pp. + last_wire_out_serial_id + )) + ); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == wa->qsx) + qs = TALER_ARL_adb->update_wire_auditor_account_progress ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + wa->section_name, + &wa->pp, + wa->in_wire_off, + wa->out_wire_off); + else + qs = TALER_ARL_adb->insert_wire_auditor_account_progress ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + wa->section_name, + &wa->pp, + wa->in_wire_off, + wa->out_wire_off); + if (0 >= qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to update auditor DB, not recording progress\n"); + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + } + GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, + &check_pending_rc, + NULL); + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsx_gwap) + qs = TALER_ARL_adb->update_wire_auditor_progress (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &pp); + else + qs = TALER_ARL_adb->insert_wire_auditor_progress (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &pp); + if (0 >= qs) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to update auditor DB, not recording progress\n"); + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + return qs; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Concluded audit step at %s\n", + GNUNET_STRINGS_absolute_time_to_string (pp.last_timestamp)); + + if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) + { + qs = TALER_ARL_edb->commit (TALER_ARL_edb->cls, + TALER_ARL_esession); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Exchange DB commit failed, rolling back transaction\n"); + TALER_ARL_adb->rollback (TALER_ARL_adb->cls, + TALER_ARL_asession); + } + else + { + qs = TALER_ARL_adb->commit (TALER_ARL_adb->cls, + TALER_ARL_asession); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Auditor DB commit failed!\n"); + } + } + } + else + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Processing failed, rolling back transaction\n"); + TALER_ARL_adb->rollback (TALER_ARL_adb->cls, + TALER_ARL_asession); + TALER_ARL_edb->rollback (TALER_ARL_edb->cls, + TALER_ARL_esession); + } + return qs; +} + + +/* ***************************** Analyze required transfers ************************ */ + +/** + * Function called on deposits that are past their due date + * and have not yet seen a wire transfer. + * + * @param cls closure + * @param rowid deposit table row of the coin's deposit + * @param coin_pub public key of the coin + * @param amount value of the deposit, including fee + * @param wire where should the funds be wired + * @param deadline what was the requested wire transfer deadline + * @param tiny did the exchange defer this transfer because it is too small? + * @param done did the exchange claim that it made a transfer? + */ +static void +wire_missing_cb (void *cls, + uint64_t rowid, + const struct TALER_CoinSpendPublicKeyP *coin_pub, + const struct TALER_Amount *amount, + const json_t *wire, + struct GNUNET_TIME_Absolute deadline, + /* bool? */ int tiny, + /* bool? */ int done) +{ + (void) cls; + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_amount_lag, + &total_amount_lag, + amount)); + if ( (GNUNET_YES == tiny) && + (0 > TALER_amount_cmp (amount, + &tiny_amount)) ) + return; /* acceptable, amount was tiny */ + TALER_ARL_report (report_lags, + json_pack ("{s:I, s:o, s:o, s:s, s:o, s:O}", + "row", (json_int_t) rowid, + "amount", TALER_JSON_from_amount (amount), + "deadline", TALER_ARL_json_from_time_abs ( + deadline), + "claimed_done", (done) ? "yes" : "no", + "coin_pub", GNUNET_JSON_from_data_auto ( + coin_pub), + "account", wire)); + +} + + +/** + * Checks that all wire transfers that should have happened + * (based on deposits) have indeed happened. + */ +static void +check_for_required_transfers () +{ + struct GNUNET_TIME_Absolute next_timestamp; + enum GNUNET_DB_QueryStatus qs; + + next_timestamp = GNUNET_TIME_absolute_get (); + (void) GNUNET_TIME_round_abs (&next_timestamp); + /* Subtract #GRACE_PERIOD, so we can be a bit behind in processing + without immediately raising undue concern */ + next_timestamp = GNUNET_TIME_absolute_subtract (next_timestamp, + GRACE_PERIOD); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing exchange's unfinished deposits (deadline: %s)\n", + GNUNET_STRINGS_absolute_time_to_string (next_timestamp)); + qs = TALER_ARL_edb->select_deposits_missing_wire (TALER_ARL_edb->cls, + TALER_ARL_esession, + pp.last_timestamp, + next_timestamp, + &wire_missing_cb, + &next_timestamp); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + pp.last_timestamp = next_timestamp; + /* conclude with success */ + commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); + GNUNET_SCHEDULER_shutdown (); +} + + +/* ***************************** Analyze reserves_out ************************ */ + +/** + * Clean up after processing wire out data. + */ +static void +conclude_wire_out () +{ + GNUNET_CONTAINER_multihashmap_destroy (out_map); + out_map = NULL; + check_for_required_transfers (); +} + + +/** + * Check that @a want is within #TIME_TOLERANCE of @a have. + * Otherwise TALER_ARL_report an inconsistency in row @a rowid of @a table. + * + * @param table where is the inconsistency (if any) + * @param rowid what is the row + * @param want what is the expected time + * @param have what is the time we got + */ +static void +check_time_difference (const char *table, + uint64_t rowid, + struct GNUNET_TIME_Absolute want, + struct GNUNET_TIME_Absolute have) +{ + struct GNUNET_TIME_Relative delta; + char *details; + + if (have.abs_value_us > want.abs_value_us) + delta = GNUNET_TIME_absolute_get_difference (want, + have); + else + delta = GNUNET_TIME_absolute_get_difference (have, + want); + if (delta.rel_value_us <= TIME_TOLERANCE.rel_value_us) + return; + + GNUNET_asprintf (&details, + "execution date mismatch (%s)", + GNUNET_STRINGS_relative_time_to_string (delta, + GNUNET_YES)); + TALER_ARL_report (report_row_minor_inconsistencies, + json_pack ("{s:s, s:I, s:s}", + "table", table, + "row", (json_int_t) rowid, + "diagnostic", details)); + GNUNET_free (details); +} + + +/** + * Function called with details about outgoing wire transfers + * as claimed by the exchange DB. + * + * @param cls a `struct WireAccount` + * @param rowid unique serial ID for the refresh session in our DB + * @param date timestamp of the transfer (roughly) + * @param wtid wire transfer subject + * @param wire wire transfer details of the receiver + * @param amount amount that was wired + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +wire_out_cb (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute date, + const struct TALER_WireTransferIdentifierRawP *wtid, + const json_t *wire, + const struct TALER_Amount *amount) +{ + struct WireAccount *wa = cls; + struct GNUNET_HashCode key; + struct ReserveOutInfo *roi; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Exchange wire OUT at %s of %s with WTID %s\n", + GNUNET_STRINGS_absolute_time_to_string (date), + TALER_amount2s (amount), + TALER_B2S (wtid)); + GNUNET_CRYPTO_hash (wtid, + sizeof (struct TALER_WireTransferIdentifierRawP), + &key); + roi = GNUNET_CONTAINER_multihashmap_get (out_map, + &key); + if (NULL == roi) + { + /* Wire transfer was not made (yet) at all (but would have been + justified), so the entire amount is missing / still to be done. + This is moderately harmless, it might just be that the aggreator + has not yet fully caught up with the transfers it should do. */ + TALER_ARL_report (report_wire_out_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", + "row", (json_int_t) rowid, + "amount_wired", TALER_JSON_from_amount (&zero), + "amount_justified", TALER_JSON_from_amount ( + amount), + "wtid", GNUNET_JSON_from_data_auto (wtid), + "timestamp", TALER_ARL_json_from_time_abs ( + date), + "diagnostic", "wire transfer not made (yet?)", + "account_section", wa->section_name)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_out_minus, + &total_bad_amount_out_minus, + amount)); + return GNUNET_OK; + } + { + char *payto_uri; + + payto_uri = TALER_JSON_wire_to_payto (wire); + if (0 != strcasecmp (payto_uri, + roi->details.credit_account_url)) + { + /* Destination bank account is wrong in actual wire transfer, so + we should count the wire transfer as entirely spurious, and + additionally consider the justified wire transfer as missing. */ + TALER_ARL_report (report_wire_out_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", + "row", (json_int_t) rowid, + "amount_wired", TALER_JSON_from_amount ( + &roi->details.amount), + "amount_justified", TALER_JSON_from_amount ( + &zero), + "wtid", GNUNET_JSON_from_data_auto (wtid), + "timestamp", TALER_ARL_json_from_time_abs ( + date), + "diagnostic", "recevier account mismatch", + "account_section", wa->section_name)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_out_plus, + &total_bad_amount_out_plus, + &roi->details.amount)); + TALER_ARL_report (report_wire_out_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", + "row", (json_int_t) rowid, + "amount_wired", TALER_JSON_from_amount ( + &zero), + "amount_justified", TALER_JSON_from_amount ( + amount), + "wtid", GNUNET_JSON_from_data_auto (wtid), + "timestamp", TALER_ARL_json_from_time_abs ( + date), + "diagnostic", "receiver account mismatch", + "account_section", wa->section_name)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_out_minus, + &total_bad_amount_out_minus, + amount)); + GNUNET_free (payto_uri); + goto cleanup; + } + GNUNET_free (payto_uri); + } + if (0 != TALER_amount_cmp (&roi->details.amount, + amount)) + { + TALER_ARL_report (report_wire_out_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", + "row", (json_int_t) rowid, + "amount_justified", TALER_JSON_from_amount ( + amount), + "amount_wired", TALER_JSON_from_amount ( + &roi->details.amount), + "wtid", GNUNET_JSON_from_data_auto (wtid), + "timestamp", TALER_ARL_json_from_time_abs ( + date), + "diagnostic", "wire amount does not match", + "account_section", wa->section_name)); + if (0 < TALER_amount_cmp (amount, + &roi->details.amount)) + { + /* amount > roi->details.amount: wire transfer was smaller than it should have been */ + struct TALER_Amount delta; + + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + amount, + &roi->details.amount)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_out_minus, + &total_bad_amount_out_minus, + &delta)); + } + else + { + /* roi->details.amount < amount: wire transfer was larger than it should have been */ + struct TALER_Amount delta; + + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + &roi->details.amount, + amount)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_out_plus, + &total_bad_amount_out_plus, + &delta)); + } + goto cleanup; + } + + check_time_difference ("wire_out", + rowid, + date, + roi->details.execution_date); +cleanup: + GNUNET_assert (GNUNET_OK == + free_roi (NULL, + &key, + roi)); + wa->pp.last_wire_out_serial_id = rowid + 1; + return GNUNET_OK; +} + + +/** + * Closure for #check_rc_matches + */ +struct CheckMatchContext +{ + + /** + * Reserve operation looking for a match + */ + const struct ReserveOutInfo *roi; + + /** + * Set to #GNUNET_YES if we found a match. + */ + int found; +}; + + +/** + * Check if any of the reserve closures match the given wire transfer. + * + * @param cls a `struct CheckMatchContext` + * @param key key of @a value in #reserve_closures + * @param value a `struct ReserveClosure` + */ +static int +check_rc_matches (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct CheckMatchContext *ctx = cls; + struct ReserveClosure *rc = value; + + if ( (0 == GNUNET_memcmp (&ctx->roi->details.wtid, + &rc->wtid)) && + (0 == strcasecmp (rc->receiver_account, + ctx->roi->details.credit_account_url)) && + (0 == TALER_amount_cmp (&rc->amount, + &ctx->roi->details.amount)) ) + { + check_time_difference ("reserves_closures", + rc->rowid, + rc->execution_date, + ctx->roi->details.execution_date); + ctx->found = GNUNET_YES; + free_rc (NULL, + key, + rc); + return GNUNET_NO; + } + return GNUNET_OK; +} + + +/** + * Check whether the given transfer was justified by a reserve closure. If + * not, complain that we failed to match an entry from #out_map. This means a + * wire transfer was made without proper justification. + * + * @param cls a `struct WireAccount` + * @param key unused key + * @param value the `struct ReserveOutInfo` to TALER_ARL_report + * @return #GNUNET_OK + */ +static int +complain_out_not_found (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct WireAccount *wa = cls; + struct ReserveOutInfo *roi = value; + struct GNUNET_HashCode rkey; + struct CheckMatchContext ctx = { + .roi = roi, + .found = GNUNET_NO + }; + + (void) key; + hash_rc (roi->details.credit_account_url, + &roi->details.wtid, + &rkey); + GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, + &rkey, + &check_rc_matches, + &ctx); + if (GNUNET_YES == ctx.found) + return GNUNET_OK; + TALER_ARL_report (report_wire_out_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", + "row", (json_int_t) 0, + "amount_wired", TALER_JSON_from_amount ( + &roi->details.amount), + "amount_justified", TALER_JSON_from_amount ( + &zero), + "wtid", GNUNET_JSON_from_data_auto ( + &roi->details.wtid), + "timestamp", TALER_ARL_json_from_time_abs ( + roi->details.execution_date), + "account_section", + wa->section_name, + "diagnostic", + "justification for wire transfer not found")); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_out_plus, + &total_bad_amount_out_plus, + &roi->details.amount)); + return GNUNET_OK; +} + + +/** + * Main function for processing 'reserves_out' data. We start by going over + * the DEBIT transactions this time, and then verify that all of them are + * justified by 'reserves_out'. + * + * @param cls `struct WireAccount` with a wire account list to process + */ +static void +process_debits (void *cls); + + +/** + * Go over the "wire_out" table of the exchange and + * verify that all wire outs are in that table. + * + * @param wa wire account we are processing + */ +static void +check_exchange_wire_out (struct WireAccount *wa) +{ + enum GNUNET_DB_QueryStatus qs; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing exchange's wire OUT table for account `%s'\n", + wa->section_name); + qs = TALER_ARL_edb->select_wire_out_above_serial_id_by_account ( + TALER_ARL_edb->cls, + TALER_ARL_esession, + wa-> + section_name, + wa->pp. + last_wire_out_serial_id, + &wire_out_cb, + wa); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_CONTAINER_multihashmap_iterate (out_map, + &complain_out_not_found, + wa); + /* clean up */ + GNUNET_CONTAINER_multihashmap_iterate (out_map, + &free_roi, + NULL); + process_debits (wa->next); +} + + +/** + * This function is called for all transactions that + * are debited from the exchange's account (outgoing + * transactions). + * + * @param cls `struct WireAccount` with current wire account to process + * @param http_status_code http status of the request + * @param ec error code in case something went wrong + * @param row_off identification of the position at which we are querying + * @param details details about the wire transfer + * @param json original response in JSON format + * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration + */ +static int +history_debit_cb (void *cls, + unsigned int http_status_code, + enum TALER_ErrorCode ec, + uint64_t row_off, + const struct TALER_BANK_DebitDetails *details, + const json_t *json) +{ + struct WireAccount *wa = cls; + struct ReserveOutInfo *roi; + size_t slen; + + (void) json; + if (NULL == details) + { + wa->dhh = NULL; + if (TALER_EC_NONE != ec) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error fetching debit history of account %s: %u/%u!\n", + wa->section_name, + http_status_code, + (unsigned int) ec); + commit (GNUNET_DB_STATUS_HARD_ERROR); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return GNUNET_SYSERR; + } + check_exchange_wire_out (wa); + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing bank DEBIT at %s of %s with WTID %s\n", + GNUNET_STRINGS_absolute_time_to_string (details->execution_date), + TALER_amount2s (&details->amount), + TALER_B2S (&details->wtid)); + /* Update offset */ + wa->out_wire_off = row_off; + slen = strlen (details->credit_account_url) + 1; + roi = GNUNET_malloc (sizeof (struct ReserveOutInfo) + + slen); + GNUNET_CRYPTO_hash (&details->wtid, + sizeof (details->wtid), + &roi->subject_hash); + roi->details.amount = details->amount; + roi->details.execution_date = details->execution_date; + roi->details.wtid = details->wtid; + roi->details.credit_account_url = (const char *) &roi[1]; + memcpy (&roi[1], + details->credit_account_url, + slen); + if (GNUNET_OK != + GNUNET_CONTAINER_multihashmap_put (out_map, + &roi->subject_hash, + roi, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + char *diagnostic; + + GNUNET_asprintf (&diagnostic, + "duplicate subject hash `%s'", + TALER_B2S (&roi->subject_hash)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_wire_format_amount, + &total_wire_format_amount, + &details->amount)); + TALER_ARL_report (report_wire_format_inconsistencies, + json_pack ("{s:o, s:I, s:s}", + "amount", TALER_JSON_from_amount ( + &details->amount), + "wire_offset", (json_int_t) row_off, + "diagnostic", diagnostic)); + GNUNET_free (diagnostic); + return GNUNET_OK; + } + return GNUNET_OK; +} + + +/** + * Main function for processing 'reserves_out' data. We start by going over + * the DEBIT transactions this time, and then verify that all of them are + * justified by 'reserves_out'. + * + * @param cls `struct WireAccount` with a wire account list to process + */ +static void +process_debits (void *cls) +{ + struct WireAccount *wa = cls; + + /* skip accounts where DEBIT is not enabled */ + while ( (NULL != wa) && + (GNUNET_NO == wa->watch_debit) ) + wa = wa->next; + if (NULL == wa) + { + /* end of iteration, now check wire_out to see + if it matches #out_map */ + conclude_wire_out (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Checking bank DEBIT records of account `%s'\n", + wa->section_name); + GNUNET_assert (NULL == wa->dhh); + wa->dhh = TALER_BANK_debit_history (ctx, + &wa->auth, + wa->out_wire_off, + INT64_MAX, + &history_debit_cb, + wa); + if (NULL == wa->dhh) + { + fprintf (stderr, + "Failed to obtain bank transaction history for `%s'\n", + wa->section_name); + commit (GNUNET_DB_STATUS_HARD_ERROR); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } +} + + +/** + * Begin analyzing wire_out. + */ +static void +begin_debit_audit (void) +{ + out_map = GNUNET_CONTAINER_multihashmap_create (1024, + GNUNET_YES); + process_debits (wa_head); +} + + +/* ***************************** Analyze reserves_in ************************ */ + +/** + * Conclude the credit history check by logging entries that + * were not found and freeing resources. Then move on to + * processing debits. + */ +static void +conclude_credit_history (void) +{ + GNUNET_CONTAINER_multihashmap_destroy (in_map); + in_map = NULL; + /* credit done, now check debits */ + begin_debit_audit (); +} + + +/** + * Function called with details about incoming wire transfers + * as claimed by the exchange DB. + * + * @param cls a `struct WireAccount` we are processing + * @param rowid unique serial ID for the entry in our DB + * @param reserve_pub public key of the reserve (also the WTID) + * @param credit amount that was received + * @param sender_account_details payto://-URL of the sender's bank account + * @param wire_reference unique identifier for the wire transfer + * @param execution_date when did we receive the funds + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +reserve_in_cb (void *cls, + uint64_t rowid, + const struct + TALER_ReservePublicKeyP *reserve_pub, + const struct TALER_Amount *credit, + const char *sender_account_details, + uint64_t wire_reference, + struct GNUNET_TIME_Absolute + execution_date) +{ + struct WireAccount *wa = cls; + struct ReserveInInfo *rii; + size_t slen; + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing exchange wire IN (%llu) at %s of %s with reserve_pub %s\n", + (unsigned long long) rowid, + GNUNET_STRINGS_absolute_time_to_string (execution_date), + TALER_amount2s (credit), + TALER_B2S (reserve_pub)); + slen = strlen (sender_account_details) + 1; + rii = GNUNET_malloc (sizeof (struct ReserveInInfo) + + slen); + rii->rowid = rowid; + rii->details.amount = *credit; + rii->details.execution_date = execution_date; + rii->details.reserve_pub = *reserve_pub; + rii->details.debit_account_url = (const char *) &rii[1]; + memcpy (&rii[1], + sender_account_details, + slen); + GNUNET_CRYPTO_hash (&wire_reference, + sizeof (uint64_t), + &rii->row_off_hash); + if (GNUNET_OK != + GNUNET_CONTAINER_multihashmap_put (in_map, + &rii->row_off_hash, + rii, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + TALER_ARL_report (report_row_inconsistencies, + json_pack ("{s:s, s:I, s:o, s:s}", + "table", "reserves_in", + "row", (json_int_t) rowid, + "wire_offset_hash", + GNUNET_JSON_from_data_auto ( + &rii->row_off_hash), + "diagnostic", "duplicate wire offset")); + GNUNET_free (rii); + return GNUNET_OK; + } + wa->pp.last_reserve_in_serial_id = rowid + 1; + return GNUNET_OK; +} + + +/** + * Complain that we failed to match an entry from #in_map. + * + * @param cls a `struct WireAccount` + * @param key unused key + * @param value the `struct ReserveInInfo` to free + * @return #GNUNET_OK + */ +static int +complain_in_not_found (void *cls, + const struct GNUNET_HashCode *key, + void *value) +{ + struct WireAccount *wa = cls; + struct ReserveInInfo *rii = value; + + (void) key; + TALER_ARL_report (report_reserve_in_inconsistencies, + json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", + "row", (json_int_t) rii->rowid, + "amount_exchange_expected", + TALER_JSON_from_amount ( + &rii->details.amount), + "amount_wired", TALER_JSON_from_amount (&zero), + "reserve_pub", GNUNET_JSON_from_data_auto ( + &rii->details.reserve_pub), + "timestamp", TALER_ARL_json_from_time_abs ( + rii->details.execution_date), + "account", wa->section_name, + "diagnostic", + "incoming wire transfer claimed by exchange not found")); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_in_minus, + &total_bad_amount_in_minus, + &rii->details.amount)); + return GNUNET_OK; +} + + +/** + * Start processing the next wire account. + * Shuts down if we are done. + * + * @param cls `struct WireAccount` with a wire account list to process + */ +static void +process_credits (void *cls); + + +/** + * This function is called for all transactions that + * are credited to the exchange's account (incoming + * transactions). + * + * @param cls `struct WireAccount` we are processing + * @param http_status HTTP status returned by the bank + * @param ec error code in case something went wrong + * @param row_off identification of the position at which we are querying + * @param details details about the wire transfer + * @param json raw response + * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration + */ +static int +history_credit_cb (void *cls, + unsigned int http_status, + enum TALER_ErrorCode ec, + uint64_t row_off, + const struct TALER_BANK_CreditDetails *details, + const json_t *json) +{ + struct WireAccount *wa = cls; + struct ReserveInInfo *rii; + struct GNUNET_HashCode key; + + (void) json; + if (NULL == details) + { + wa->chh = NULL; + if (TALER_EC_NONE != ec) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Error fetching credit history of account %s: %u/%u!\n", + wa->section_name, + http_status, + (unsigned int) ec); + commit (GNUNET_DB_STATUS_HARD_ERROR); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return GNUNET_SYSERR; + } + /* end of operation */ + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Reconciling CREDIT processing of account `%s'\n", + wa->section_name); + GNUNET_CONTAINER_multihashmap_iterate (in_map, + &complain_in_not_found, + wa); + /* clean up before 2nd phase */ + GNUNET_CONTAINER_multihashmap_iterate (in_map, + &free_rii, + NULL); + process_credits (wa->next); + return GNUNET_OK; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing bank CREDIT at %s of %s with Reserve-pub %s\n", + GNUNET_STRINGS_absolute_time_to_string (details->execution_date), + TALER_amount2s (&details->amount), + TALER_B2S (&details->reserve_pub)); + GNUNET_CRYPTO_hash (&row_off, + sizeof (row_off), + &key); + rii = GNUNET_CONTAINER_multihashmap_get (in_map, + &key); + if (NULL == rii) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Failed to find wire transfer at `%s' in exchange database. Audit ends at this point in time.\n", + GNUNET_STRINGS_absolute_time_to_string ( + details->execution_date)); + wa->chh = NULL; + process_credits (wa->next); + return GNUNET_SYSERR; /* not an error, just end of processing */ + } + + /* Update offset */ + wa->in_wire_off = row_off; + /* compare records with expected data */ + if (0 != GNUNET_memcmp (&details->reserve_pub, + &rii->details.reserve_pub)) + { + TALER_ARL_report (report_reserve_in_inconsistencies, + json_pack ("{s:I, s:I, s:o, s:o, s:o, s:o, s:s}", + "row", (json_int_t) rii->rowid, + "bank_row", (json_int_t) row_off, + "amount_exchange_expected", + TALER_JSON_from_amount ( + &rii->details.amount), + "amount_wired", TALER_JSON_from_amount (&zero), + "reserve_pub", GNUNET_JSON_from_data_auto ( + &rii->details.reserve_pub), + "timestamp", TALER_ARL_json_from_time_abs ( + rii->details.execution_date), + "diagnostic", "wire subject does not match")); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_in_minus, + &total_bad_amount_in_minus, + &rii->details.amount)); + TALER_ARL_report (report_reserve_in_inconsistencies, + json_pack ("{s:I, s:I, s:o, s:o, s:o, s:o, s:s}", + "row", (json_int_t) rii->rowid, + "bank_row", (json_int_t) row_off, + "amount_exchange_expected", + TALER_JSON_from_amount ( + &zero), + "amount_wired", TALER_JSON_from_amount ( + &details->amount), + "reserve_pub", GNUNET_JSON_from_data_auto ( + &details->reserve_pub), + "timestamp", TALER_ARL_json_from_time_abs ( + details->execution_date), + "diagnostic", "wire subject does not match")); + + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_in_plus, + &total_bad_amount_in_plus, + &details->amount)); + goto cleanup; + } + if (0 != TALER_amount_cmp (&rii->details.amount, + &details->amount)) + { + TALER_ARL_report (report_reserve_in_inconsistencies, + json_pack ("{s:I, s:I, s:o, s:o, s:o, s:o, s:s}", + "row", (json_int_t) rii->rowid, + "bank_row", (json_int_t) row_off, + "amount_exchange_expected", + TALER_JSON_from_amount ( + &rii->details.amount), + "amount_wired", TALER_JSON_from_amount ( + &details->amount), + "reserve_pub", GNUNET_JSON_from_data_auto ( + &details->reserve_pub), + "timestamp", TALER_ARL_json_from_time_abs ( + details->execution_date), + "diagnostic", "wire amount does not match")); + if (0 < TALER_amount_cmp (&details->amount, + &rii->details.amount)) + { + /* details->amount > rii->details.amount: wire transfer was larger than it should have been */ + struct TALER_Amount delta; + + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + &details->amount, + &rii->details.amount)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_in_plus, + &total_bad_amount_in_plus, + &delta)); + } + else + { + /* rii->details.amount < details->amount: wire transfer was smaller than it should have been */ + struct TALER_Amount delta; + + GNUNET_break (GNUNET_OK == + TALER_amount_subtract (&delta, + &rii->details.amount, + &details->amount)); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_bad_amount_in_minus, + &total_bad_amount_in_minus, + &delta)); + } + goto cleanup; + } + if (0 != strcasecmp (details->debit_account_url, + rii->details.debit_account_url)) + { + TALER_ARL_report (report_missattribution_in_inconsistencies, + json_pack ("{s:o, s:I, s:I, s:o}", + "amount", TALER_JSON_from_amount ( + &rii->details.amount), + "row", (json_int_t) rii->rowid, + "bank_row", (json_int_t) row_off, + "reserve_pub", GNUNET_JSON_from_data_auto ( + &rii->details.reserve_pub))); + GNUNET_break (GNUNET_OK == + TALER_amount_add (&total_missattribution_in, + &total_missattribution_in, + &rii->details.amount)); + } + if (details->execution_date.abs_value_us != + rii->details.execution_date.abs_value_us) + { + TALER_ARL_report (report_row_minor_inconsistencies, + json_pack ("{s:s, s:I, s:I, s:s}", + "table", "reserves_in", + "row", (json_int_t) rii->rowid, + "bank_row", (json_int_t) row_off, + "diagnostic", "execution date mismatch")); + } +cleanup: + GNUNET_assert (GNUNET_OK == + free_rii (NULL, + &key, + rii)); + return GNUNET_OK; +} + + +/* ***************************** Setup logic ************************ */ + + +/** + * Start processing the next wire account. + * Shuts down if we are done. + * + * @param cls `struct WireAccount` with a wire account list to process + */ +static void +process_credits (void *cls) +{ + struct WireAccount *wa = cls; + enum GNUNET_DB_QueryStatus qs; + + /* skip accounts where CREDIT is not enabled */ + while ( (NULL != wa) && + (GNUNET_NO == wa->watch_credit) ) + wa = wa->next; + if (NULL == wa) + { + /* done with all accounts, conclude check */ + conclude_credit_history (); + return; + } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Analyzing exchange's wire IN table for account `%s'\n", + wa->section_name); + qs = TALER_ARL_edb->select_reserves_in_above_serial_id_by_account ( + TALER_ARL_edb->cls, + TALER_ARL_esession, + wa-> + section_name, + wa->pp. + last_reserve_in_serial_id, + & + reserve_in_cb, + wa); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting bank CREDIT history of account `%s'\n", + wa->section_name); + wa->chh = TALER_BANK_credit_history (ctx, + &wa->auth, + wa->in_wire_off, + INT64_MAX, + &history_credit_cb, + wa); + if (NULL == wa->chh) + { + fprintf (stderr, + "Failed to obtain bank transaction history\n"); + commit (GNUNET_DB_STATUS_HARD_ERROR); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } +} + + +/** + * Begin audit of CREDITs to the exchange. + */ +static void +begin_credit_audit (void) +{ + in_map = GNUNET_CONTAINER_multihashmap_create (1024, + GNUNET_YES); + /* now go over all bank accounts and check delta with in_map */ + process_credits (wa_head); +} + + +/** + * Function called about reserve closing operations + * the aggregator triggered. + * + * @param cls closure + * @param rowid row identifier used to uniquely identify the reserve closing operation + * @param execution_date when did we execute the close operation + * @param amount_with_fee how much did we debit the reserve + * @param closing_fee how much did we charge for closing the reserve + * @param reserve_pub public key of the reserve + * @param receiver_account where did we send the funds, in payto://-format + * @param wtid identifier used for the wire transfer + * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop + */ +static int +reserve_closed_cb (void *cls, + uint64_t rowid, + struct GNUNET_TIME_Absolute execution_date, + const struct TALER_Amount *amount_with_fee, + const struct TALER_Amount *closing_fee, + const struct TALER_ReservePublicKeyP *reserve_pub, + const char *receiver_account, + const struct TALER_WireTransferIdentifierRawP *wtid) +{ + struct ReserveClosure *rc; + struct GNUNET_HashCode key; + + (void) cls; + rc = GNUNET_new (struct ReserveClosure); + if (GNUNET_SYSERR == + TALER_amount_subtract (&rc->amount, + amount_with_fee, + closing_fee)) + { + TALER_ARL_report (report_row_inconsistencies, + json_pack ("{s:s, s:I, s:o, s:o, s:o, s:s}", + "table", "reserves_closures", + "row", (json_int_t) rowid, + "reserve_pub", GNUNET_JSON_from_data_auto ( + reserve_pub), + "amount_with_fee", TALER_JSON_from_amount ( + amount_with_fee), + "closing_fee", TALER_JSON_from_amount ( + closing_fee), + "diagnostic", + "closing fee above total amount")); + GNUNET_free (rc); + return GNUNET_OK; + } + pp.last_reserve_close_uuid + = GNUNET_MAX (pp.last_reserve_close_uuid, + rowid + 1); + rc->receiver_account = GNUNET_strdup (receiver_account); + rc->wtid = *wtid; + rc->execution_date = execution_date; + rc->rowid = rowid; + hash_rc (receiver_account, + wtid, + &key); + (void) GNUNET_CONTAINER_multihashmap_put (reserve_closures, + &key, + rc, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); + return GNUNET_OK; +} + + +/** + * Start the database transactions and begin the audit. + * + * @param cls NULL + * @return transaction status code + */ +static enum GNUNET_DB_QueryStatus +begin_transaction (void *cls) +{ + int ret; + + (void) cls; + ret = TALER_ARL_adb->start (TALER_ARL_adb->cls, + TALER_ARL_asession); + if (GNUNET_OK != ret) + { + GNUNET_break (0); + return GNUNET_DB_STATUS_HARD_ERROR; + } + TALER_ARL_edb->preflight (TALER_ARL_edb->cls, + TALER_ARL_esession); + ret = TALER_ARL_edb->start (TALER_ARL_edb->cls, + TALER_ARL_esession, + "wire auditor"); + if (GNUNET_OK != ret) + { + GNUNET_break (0); + return GNUNET_DB_STATUS_HARD_ERROR; + } + for (struct WireAccount *wa = wa_head; + NULL != wa; + wa = wa->next) + { + wa->qsx = TALER_ARL_adb->get_wire_auditor_account_progress ( + TALER_ARL_adb->cls, + TALER_ARL_asession, + & + TALER_ARL_master_pub, + wa->section_name, + &wa->pp, + &wa->in_wire_off, + &wa-> + out_wire_off); + if (0 > wa->qsx) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == wa->qsx); + return GNUNET_DB_STATUS_HARD_ERROR; + } + wa->start_pp = wa->pp; + } + qsx_gwap = TALER_ARL_adb->get_wire_auditor_progress (TALER_ARL_adb->cls, + TALER_ARL_asession, + &TALER_ARL_master_pub, + &pp); + if (0 > qsx_gwap) + { + GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx_gwap); + return GNUNET_DB_STATUS_HARD_ERROR; + } + if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsx_gwap) + { + GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, + _ ( + "First analysis using this auditor, starting audit from scratch\n")); + } + else + { + start_pp = pp; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Resuming audit at %s / %llu\n", + GNUNET_STRINGS_absolute_time_to_string (pp.last_timestamp), + (unsigned long long) pp.last_reserve_close_uuid); + } + + { + enum GNUNET_DB_QueryStatus qs; + + qs = TALER_ARL_edb->select_reserve_closed_above_serial_id ( + TALER_ARL_edb->cls, + TALER_ARL_esession, + pp. + last_reserve_close_uuid, + & + reserve_closed_cb, + NULL); + if (0 > qs) + { + GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); + return GNUNET_DB_STATUS_HARD_ERROR; + } + } + begin_credit_audit (); + return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; +} + + +/** + * Function called with information about a wire account. Adds the + * account to our list for processing (if it is enabled and we can + * load the plugin). + * + * @param cls closure, NULL + * @param ai account information + */ +static void +process_account_cb (void *cls, + const struct TALER_EXCHANGEDB_AccountInfo *ai) +{ + struct WireAccount *wa; + + (void) cls; + if ( (GNUNET_NO == ai->debit_enabled) && + (GNUNET_NO == ai->credit_enabled) ) + return; /* not an active exchange account */ + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Found exchange account `%s'\n", + ai->section_name); + wa = GNUNET_new (struct WireAccount); + wa->section_name = GNUNET_strdup (ai->section_name); + wa->watch_debit = ai->debit_enabled; + wa->watch_credit = ai->credit_enabled; + if (GNUNET_OK != + TALER_BANK_auth_parse_cfg (TALER_ARL_cfg, + ai->section_name, + &wa->auth)) + { + GNUNET_break (0); + GNUNET_free (wa->section_name); + GNUNET_free (wa); + + fprintf (stderr, + "Failed to access bank account `%s'\n", + wa->section_name); + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_CONTAINER_DLL_insert (wa_head, + wa_tail, + wa); +} + + +/** + * Main function that will be run. + * + * @param cls closure + * @param args remaining command-line arguments + * @param TALER_ARL_cfgfile name of the configuration file used (for saving, can be NULL!) + * @param c configuration + */ +static void +run (void *cls, + char *const *args, + const char *TALER_ARL_cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) +{ + (void) cls; + (void) args; + (void) TALER_ARL_cfgfile; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Launching auditor\n"); + if (GNUNET_OK != + TALER_ARL_init (c)) + { + global_ret = 1; + return; + } + if (GNUNET_OK != + TALER_config_get_amount (TALER_ARL_cfg, + "auditor", + "TINY_AMOUNT", + &tiny_amount)) + { + GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, + "auditor", + "TINY_AMOUNT", + "invalid amount"); + global_ret = 1; + return; + } + GNUNET_SCHEDULER_add_shutdown (&do_shutdown, + NULL); + ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, + &rc); + rc = GNUNET_CURL_gnunet_rc_create (ctx); + if (NULL == ctx) + { + GNUNET_break (0); + return; + } + reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, + GNUNET_NO); + GNUNET_assert (NULL != + (report_wire_out_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_reserve_in_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_row_minor_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_wire_format_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_row_inconsistencies = json_array ())); + GNUNET_assert (NULL != + (report_missattribution_in_inconsistencies = + json_array ())); + GNUNET_assert (NULL != + (report_lags = json_array ())); + GNUNET_assert (NULL != + (report_closure_lags = json_array ())); + GNUNET_assert (NULL != + (report_account_progress = json_array ())); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_amount_out_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_amount_out_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_amount_in_plus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_bad_amount_in_minus)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_missattribution_in)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_amount_lag)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_closure_amount_lag)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &total_wire_format_amount)); + GNUNET_assert (GNUNET_OK == + TALER_amount_get_zero (TALER_ARL_currency, + &zero)); + TALER_EXCHANGEDB_find_accounts (TALER_ARL_cfg, + &process_account_cb, + NULL); + if (GNUNET_OK != + TALER_ARL_setup_sessions_and_run (&begin_transaction, + NULL)) + { + global_ret = 1; + GNUNET_SCHEDULER_shutdown (); + } +} + + +/** + * The main function of the database initialization tool. + * Used to initialize the Taler Exchange's database. + * + * @param argc number of arguments from the command line + * @param argv command line arguments + * @return 0 ok, 1 on error + */ +int +main (int argc, + char *const *argv) +{ + const struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_option_base32_auto ('m', + "exchange-key", + "KEY", + "public key of the exchange (Crockford base32 encoded)", + &TALER_ARL_master_pub), + GNUNET_GETOPT_option_flag ('r', + "TALER_ARL_restart", + "TALER_ARL_restart audit from the beginning (required on first run)", + &TALER_ARL_restart), + GNUNET_GETOPT_option_timetravel ('T', + "timetravel"), + GNUNET_GETOPT_OPTION_END + }; + + /* force linker to link against libtalerutil; if we do + not do this, the linker may "optimize" libtalerutil + away and skip #TALER_OS_init(), which we do need */ + (void) TALER_project_data_default (); + GNUNET_assert (GNUNET_OK == + GNUNET_log_setup ("taler-wire-auditor", + "MESSAGE", + NULL)); + if (GNUNET_OK != + GNUNET_PROGRAM_run (argc, + argv, + "taler-wire-auditor", + "Audit exchange database for consistency with the bank's wire transfers", + options, + &run, + NULL)) + return 1; + return global_ret; +} + + +/* end of taler-wire-auditor.c */ diff --git a/src/auditor/taler-wire-auditor.c b/src/auditor/taler-wire-auditor.c deleted file mode 100644 index fb8e52f7b..000000000 --- a/src/auditor/taler-wire-auditor.c +++ /dev/null @@ -1,2354 +0,0 @@ -/* - This file is part of TALER - Copyright (C) 2017-2020 Taler Systems SA - - TALER is free software; you can redistribute it and/or modify it under the - terms of the GNU General Public License as published by the Free Software - Foundation; either version 3, or (at your option) any later version. - - TALER is distributed in the hope that it will be useful, but WITHOUT ANY - WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR - A PARTICULAR PURPOSE. See the GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along with - TALER; see the file COPYING. If not, see -*/ -/** - * @file auditor/taler-wire-auditor.c - * @brief audits that wire transfers match those from an exchange database. - * @author Christian Grothoff - * - * - First, this auditor verifies that 'reserves_in' actually matches - * the incoming wire transfers from the bank. - * - Second, we check that the outgoing wire transfers match those - * given in the 'wire_out' and 'reserve_closures' tables - * - Finally, we check that all wire transfers that should have been made, - * were actually made - */ -#include "platform.h" -#include -#include -#include "taler_auditordb_plugin.h" -#include "taler_exchangedb_lib.h" -#include "taler_json_lib.h" -#include "taler_bank_service.h" -#include "taler_signatures.h" - -/** - * How much time do we allow the aggregator to lag behind? If - * wire transfers should have been made more than #GRACE_PERIOD - * before, we issue warnings. - */ -#define GRACE_PERIOD GNUNET_TIME_UNIT_HOURS - -/** - * How much do we allow the bank and the exchange to disagree about - * timestamps? Should be sufficiently large to avoid bogus reports from deltas - * created by imperfect clock synchronization and network delay. - */ -#define TIME_TOLERANCE GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, \ - 15) - - -/** - * Information we keep for each supported account. - */ -struct WireAccount -{ - /** - * Accounts are kept in a DLL. - */ - struct WireAccount *next; - - /** - * Plugins are kept in a DLL. - */ - struct WireAccount *prev; - - /** - * Authentication data for the account. - */ - struct TALER_BANK_AuthenticationData auth; - - /** - * Name of the section that configures this account. - */ - char *section_name; - - /** - * Active wire request for the transaction history. - */ - struct TALER_BANK_CreditHistoryHandle *chh; - - /** - * Active wire request for the transaction history. - */ - struct TALER_BANK_DebitHistoryHandle *dhh; - - /** - * Progress point for this account. - */ - struct TALER_AUDITORDB_WireAccountProgressPoint pp; - - /** - * Initial progress point for this account. - */ - struct TALER_AUDITORDB_WireAccountProgressPoint start_pp; - - /** - * Where we are in the inbound (CREDIT) transaction history. - */ - uint64_t in_wire_off; - - /** - * Where we are in the inbound (DEBIT) transaction history. - */ - uint64_t out_wire_off; - - /** - * We should check for inbound transactions to this account. - */ - int watch_credit; - - /** - * We should check for outbound transactions from this account. - */ - int watch_debit; - - /** - * Return value when we got this account's progress point. - */ - enum GNUNET_DB_QueryStatus qsx; -}; - - -/** - * Information we track for a reserve being closed. - */ -struct ReserveClosure -{ - /** - * Row in the reserves_closed table for this action. - */ - uint64_t rowid; - - /** - * When was the reserve closed? - */ - struct GNUNET_TIME_Absolute execution_date; - - /** - * Amount transferred (amount remaining minus fee). - */ - struct TALER_Amount amount; - - /** - * Target account where the money was sent. - */ - char *receiver_account; - - /** - * Wire transfer subject used. - */ - struct TALER_WireTransferIdentifierRawP wtid; -}; - - -/** - * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. - */ -static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; - -/** - * Return value from main(). - */ -static int global_ret; - -/** - * Command-line option "-r": restart audit from scratch - */ -static int restart; - -/** - * Time when we started the wire audit. - */ -static struct GNUNET_TIME_Absolute start_time; - -/** - * Handle to access the exchange's database. - */ -static struct TALER_EXCHANGEDB_Plugin *edb; - -/** - * Which currency are we doing the audit for? - */ -static char *currency; - -/** - * Our configuration. - */ -static const struct GNUNET_CONFIGURATION_Handle *cfg; - -/** - * Map with information about incoming wire transfers. - * Maps hashes of the wire offsets to `struct ReserveInInfo`s. - */ -static struct GNUNET_CONTAINER_MultiHashMap *in_map; - -/** - * Map with information about outgoing wire transfers. - * Maps hashes of the wire subjects (in binary encoding) - * to `struct ReserveOutInfo`s. - */ -static struct GNUNET_CONTAINER_MultiHashMap *out_map; - -/** - * Our session with the #edb. - */ -static struct TALER_EXCHANGEDB_Session *esession; - -/** - * Handle to access the auditor's database. - */ -static struct TALER_AUDITORDB_Plugin *adb; - -/** - * Our session with the #adb. - */ -static struct TALER_AUDITORDB_Session *asession; - -/** - * Master public key of the exchange to audit. - */ -static struct TALER_MasterPublicKeyP master_pub; - -/** - * Head of list of wire accounts we still need to look at. - */ -static struct WireAccount *wa_head; - -/** - * Tail of list of wire accounts we still need to look at. - */ -static struct WireAccount *wa_tail; - -/** - * Query status for the incremental processing status in the auditordb. - * Return value from our call to the "get_wire_auditor_progress" function. - */ -static enum GNUNET_DB_QueryStatus qsx_gwap; - -/** - * Last reserve_in / wire_out serial IDs seen. - */ -static struct TALER_AUDITORDB_WireProgressPoint pp; - -/** - * Last reserve_in / wire_out serial IDs seen. - */ -static struct TALER_AUDITORDB_WireProgressPoint start_pp; - -/** - * Array of reports about row inconsitencies in wire_out table. - */ -static json_t *report_wire_out_inconsistencies; - -/** - * Array of reports about row inconsitencies in reserves_in table. - */ -static json_t *report_reserve_in_inconsistencies; - -/** - * Array of reports about wrong bank account being recorded for - * incoming wire transfers. - */ -static json_t *report_missattribution_in_inconsistencies; - -/** - * Array of reports about row inconcistencies. - */ -static json_t *report_row_inconsistencies; - -/** - * Array of reports about inconcistencies in the database about - * the incoming wire transfers (exchange is not exactly to blame). - */ -static json_t *report_wire_format_inconsistencies; - -/** - * Array of reports about minor row inconcistencies. - */ -static json_t *report_row_minor_inconsistencies; - -/** - * Array of reports about lagging transactions from deposits. - */ -static json_t *report_lags; - -/** - * Array of reports about lagging transactions from reserve closures. - */ -static json_t *report_closure_lags; - -/** - * Array of per-account progress data. - */ -static json_t *report_account_progress; - -/** - * Amount that is considered "tiny" - */ -static struct TALER_Amount tiny_amount; - -/** - * Total amount that was transferred too much from the exchange. - */ -static struct TALER_Amount total_bad_amount_out_plus; - -/** - * Total amount that was transferred too little from the exchange. - */ -static struct TALER_Amount total_bad_amount_out_minus; - -/** - * Total amount that was transferred too much to the exchange. - */ -static struct TALER_Amount total_bad_amount_in_plus; - -/** - * Total amount that was transferred too little to the exchange. - */ -static struct TALER_Amount total_bad_amount_in_minus; - -/** - * Total amount where the exchange has the wrong sender account - * for incoming funds and may thus wire funds to the wrong - * destination when closing the reserve. - */ -static struct TALER_Amount total_missattribution_in; - -/** - * Total amount which the exchange did not transfer in time. - */ -static struct TALER_Amount total_amount_lag; - -/** - * Total amount of reserve closures which the exchange did not transfer in time. - */ -static struct TALER_Amount total_closure_amount_lag; - -/** - * Total amount affected by wire format trouble.s - */ -static struct TALER_Amount total_wire_format_amount; - -/** - * Amount of zero in our currency. - */ -static struct TALER_Amount zero; - -/** - * Handle to the context for interacting with the bank. - */ -static struct GNUNET_CURL_Context *ctx; - -/** - * Scheduler context for running the @e ctx. - */ -static struct GNUNET_CURL_RescheduleContext *rc; - - -/* ***************************** Shutdown **************************** */ - -/** - * Entry in map with wire information we expect to obtain from the - * bank later. - */ -struct ReserveInInfo -{ - - /** - * Hash of expected row offset. - */ - struct GNUNET_HashCode row_off_hash; - - /** - * Expected details about the wire transfer. - * The member "account_url" is to be allocated - * at the end of this struct! - */ - struct TALER_BANK_CreditDetails details; - - /** - * RowID in reserves_in table. - */ - uint64_t rowid; - -}; - - -/** - * Entry in map with wire information we expect to obtain from the - * #edb later. - */ -struct ReserveOutInfo -{ - - /** - * Hash of the wire transfer subject. - */ - struct GNUNET_HashCode subject_hash; - - /** - * Expected details about the wire transfer. - */ - struct TALER_BANK_DebitDetails details; - -}; - - -/** - * Convert absolute time to human-readable JSON string. - * - * @param at time to convert - * @return human-readable string representing the time - */ -static json_t * -json_from_time_abs (struct GNUNET_TIME_Absolute at) -{ - return json_string - (GNUNET_STRINGS_absolute_time_to_string (at)); -} - - -/** - * Free entry in #in_map. - * - * @param cls NULL - * @param key unused key - * @param value the `struct ReserveInInfo` to free - * @return #GNUNET_OK - */ -static int -free_rii (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ReserveInInfo *rii = value; - - (void) cls; - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (in_map, - key, - rii)); - GNUNET_free (rii); - return GNUNET_OK; -} - - -/** - * Free entry in #out_map. - * - * @param cls NULL - * @param key unused key - * @param value the `struct ReserveOutInfo` to free - * @return #GNUNET_OK - */ -static int -free_roi (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ReserveOutInfo *roi = value; - - (void) cls; - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (out_map, - key, - roi)); - GNUNET_free (roi); - return GNUNET_OK; -} - - -/** - * Free entry in #reserve_closures. - * - * @param cls NULL - * @param key unused key - * @param value the `struct ReserveClosure` to free - * @return #GNUNET_OK - */ -static int -free_rc (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ReserveClosure *rc = value; - - (void) cls; - GNUNET_assert (GNUNET_YES == - GNUNET_CONTAINER_multihashmap_remove (reserve_closures, - key, - rc)); - GNUNET_free (rc->receiver_account); - GNUNET_free (rc); - return GNUNET_OK; -} - - -/** - * Task run on shutdown. - * - * @param cls NULL - */ -static void -do_shutdown (void *cls) -{ - struct WireAccount *wa; - - (void) cls; - if (NULL != ctx) - { - GNUNET_CURL_fini (ctx); - ctx = NULL; - } - if (NULL != rc) - { - GNUNET_CURL_gnunet_rc_destroy (rc); - rc = NULL; - } - - if (NULL != report_row_inconsistencies) - { - json_t *report; - - GNUNET_assert (NULL != report_row_minor_inconsistencies); - report = json_pack ("{s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:o, s:o," - " s:o, s:o, s:o, s:I, s:I," - " s:o, s:o, s:o }", - /* blocks of 5 */ - /* Tested in test-auditor.sh #11, #15, #20 */ - "wire_out_amount_inconsistencies", - report_wire_out_inconsistencies, - "total_wire_out_delta_plus", - TALER_JSON_from_amount (&total_bad_amount_out_plus), - /* Tested in test-auditor.sh #11, #15, #19 */ - "total_wire_out_delta_minus", - TALER_JSON_from_amount (&total_bad_amount_out_minus), - /* Tested in test-auditor.sh #2 */ - "reserve_in_amount_inconsistencies", - report_reserve_in_inconsistencies, - /* Tested in test-auditor.sh #2 */ - "total_wire_in_delta_plus", - TALER_JSON_from_amount (&total_bad_amount_in_plus), - /* block */ - /* Tested in test-auditor.sh #3 */ - "total_wire_in_delta_minus", - TALER_JSON_from_amount (&total_bad_amount_in_minus), - /* Tested in test-auditor.sh #9 */ - "missattribution_in_inconsistencies", - report_missattribution_in_inconsistencies, - /* Tested in test-auditor.sh #9 */ - "total_missattribution_in", - TALER_JSON_from_amount (&total_missattribution_in), - "row_inconsistencies", - report_row_inconsistencies, - /* Tested in test-auditor.sh #10/#17 */ - "row_minor_inconsistencies", - report_row_minor_inconsistencies, - /* block */ - /* Tested in test-auditor.sh #19 */ - "total_wire_format_amount", - TALER_JSON_from_amount (&total_wire_format_amount), - /* Tested in test-auditor.sh #19 */ - "wire_format_inconsistencies", - report_wire_format_inconsistencies, - /* Tested in test-auditor.sh #1 */ - "total_amount_lag", - TALER_JSON_from_amount (&total_amount_lag), - /* Tested in test-auditor.sh #1 */ - "lag_details", - report_lags, - /* Tested in test-auditor.sh #22 */ - "total_closure_amount_lag", - TALER_JSON_from_amount (&total_closure_amount_lag), - /* blocks of 5 */ - /* Tested in test-auditor.sh #22 */ - "reserve_lag_details", - report_closure_lags, - "wire_auditor_start_time", - json_from_time_abs (start_time), - "wire_auditor_end_time", - json_from_time_abs (GNUNET_TIME_absolute_get ()), - "start_pp_reserve_close_uuid", - (json_int_t) start_pp.last_reserve_close_uuid, - "end_pp_reserve_close_uuid", - (json_int_t) pp.last_reserve_close_uuid, - /* blocks of 5 */ - "start_pp_last_timestamp", - json_from_time_abs (start_pp.last_timestamp), - "end_pp_last_timestamp", - json_from_time_abs (pp.last_timestamp), - "account_progress", - report_account_progress - ); - GNUNET_break (NULL != report); - json_dumpf (report, - stdout, - JSON_INDENT (2)); - json_decref (report); - report_wire_out_inconsistencies = NULL; - report_reserve_in_inconsistencies = NULL; - report_row_inconsistencies = NULL; - report_row_minor_inconsistencies = NULL; - report_missattribution_in_inconsistencies = NULL; - report_lags = NULL; - report_closure_lags = NULL; - report_account_progress = NULL; - report_wire_format_inconsistencies = NULL; - } - if (NULL != reserve_closures) - { - GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, - &free_rc, - NULL); - GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); - reserve_closures = NULL; - } - if (NULL != in_map) - { - GNUNET_CONTAINER_multihashmap_iterate (in_map, - &free_rii, - NULL); - GNUNET_CONTAINER_multihashmap_destroy (in_map); - in_map = NULL; - } - if (NULL != out_map) - { - GNUNET_CONTAINER_multihashmap_iterate (out_map, - &free_roi, - NULL); - GNUNET_CONTAINER_multihashmap_destroy (out_map); - out_map = NULL; - } - while (NULL != (wa = wa_head)) - { - if (NULL != wa->dhh) - { - TALER_BANK_debit_history_cancel (wa->dhh); - wa->dhh = NULL; - } - if (NULL != wa->chh) - { - TALER_BANK_credit_history_cancel (wa->chh); - wa->chh = NULL; - } - GNUNET_CONTAINER_DLL_remove (wa_head, - wa_tail, - wa); - TALER_BANK_auth_free (&wa->auth); - GNUNET_free (wa->section_name); - GNUNET_free (wa); - } - if (NULL != adb) - { - TALER_AUDITORDB_plugin_unload (adb); - adb = NULL; - } - if (NULL != edb) - { - TALER_EXCHANGEDB_plugin_unload (edb); - edb = NULL; - } -} - - -/* ***************************** Report logic **************************** */ - - -/** - * Add @a object to the report @a array. Fail hard if this fails. - * - * @param array report array to append @a object to - * @param object object to append, should be check that it is not NULL - */ -static void -report (json_t *array, - json_t *object) -{ - GNUNET_assert (NULL != object); - GNUNET_assert (0 == - json_array_append_new (array, - object)); -} - - -/** - * Detect any entries in #reserve_closures that were not yet - * observed on the wire transfer side and update the progress - * point accordingly. - * - * @param cls NULL - * @param key unused key - * @param value the `struct ReserveClosure` to free - * @return #GNUNET_OK - */ -static int -check_pending_rc (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct ReserveClosure *rc = value; - - (void) cls; - (void) key; - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_closure_amount_lag, - &total_closure_amount_lag, - &rc->amount)); - if ( (0 != rc->amount.value) || - (0 != rc->amount.fraction) ) - report (report_closure_lags, - json_pack ("{s:I, s:o, s:o, s:o, s:s}", - "row", (json_int_t) rc->rowid, - "amount", TALER_JSON_from_amount (&rc->amount), - "deadline", json_from_time_abs (rc->execution_date), - "wtid", GNUNET_JSON_from_data_auto (&rc->wtid), - "account", rc->receiver_account)); - pp.last_reserve_close_uuid - = GNUNET_MIN (pp.last_reserve_close_uuid, - rc->rowid); - return GNUNET_OK; -} - - -/** - * Compute the key under which a reserve closure for a given - * @a receiver_account and @a wtid would be stored. - * - * @param receiver_account payto://-URI of the account - * @param wtid wire transfer identifier used - * @param[out] key set to the key - */ -static void -hash_rc (const char *receiver_account, - const struct TALER_WireTransferIdentifierRawP *wtid, - struct GNUNET_HashCode *key) -{ - size_t slen = strlen (receiver_account); - char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; - - memcpy (buf, - wtid, - sizeof (*wtid)); - memcpy (&buf[sizeof (*wtid)], - receiver_account, - slen); - GNUNET_CRYPTO_hash (buf, - sizeof (buf), - key); -} - - -/* *************************** General transaction logic ****************** */ - -/** - * Commit the transaction, checkpointing our progress in the auditor - * DB. - * - * @param qs transaction status so far - * @return transaction status code - */ -static enum GNUNET_DB_QueryStatus -commit (enum GNUNET_DB_QueryStatus qs) -{ - if (0 > qs) - { - if (GNUNET_DB_STATUS_SOFT_ERROR == qs) - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Serialization issue, not recording progress\n"); - else - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Hard error, not recording progress\n"); - adb->rollback (adb->cls, - asession); - edb->rollback (edb->cls, - esession); - return qs; - } - for (struct WireAccount *wa = wa_head; - NULL != wa; - wa = wa->next) - { - GNUNET_assert (0 == - json_array_append_new (report_account_progress, - json_pack ( - "{s:s, s:I, s:I, s:I, s:I}", - "account", - wa->section_name, - "start_reserve_in", - (json_int_t) wa->start_pp. - last_reserve_in_serial_id, - "end_reserve_in", - (json_int_t) wa->pp. - last_reserve_in_serial_id, - "start_wire_out", - (json_int_t) wa->start_pp. - last_wire_out_serial_id, - "end_wire_out", - (json_int_t) wa->pp. - last_wire_out_serial_id - )) - ); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == wa->qsx) - qs = adb->update_wire_auditor_account_progress (adb->cls, - asession, - &master_pub, - wa->section_name, - &wa->pp, - wa->in_wire_off, - wa->out_wire_off); - else - qs = adb->insert_wire_auditor_account_progress (adb->cls, - asession, - &master_pub, - wa->section_name, - &wa->pp, - wa->in_wire_off, - wa->out_wire_off); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to update auditor DB, not recording progress\n"); - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - } - GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, - &check_pending_rc, - NULL); - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qsx_gwap) - qs = adb->update_wire_auditor_progress (adb->cls, - asession, - &master_pub, - &pp); - else - qs = adb->insert_wire_auditor_progress (adb->cls, - asession, - &master_pub, - &pp); - if (0 >= qs) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to update auditor DB, not recording progress\n"); - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - return qs; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Concluded audit step at %s\n", - GNUNET_STRINGS_absolute_time_to_string (pp.last_timestamp)); - - if (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT == qs) - { - qs = edb->commit (edb->cls, - esession); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Exchange DB commit failed, rolling back transaction\n"); - adb->rollback (adb->cls, - asession); - } - else - { - qs = adb->commit (adb->cls, - asession); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Auditor DB commit failed!\n"); - } - } - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Processing failed, rolling back transaction\n"); - adb->rollback (adb->cls, - asession); - edb->rollback (edb->cls, - esession); - } - return qs; -} - - -/* ***************************** Analyze required transfers ************************ */ - -/** - * Function called on deposits that are past their due date - * and have not yet seen a wire transfer. - * - * @param cls closure - * @param rowid deposit table row of the coin's deposit - * @param coin_pub public key of the coin - * @param amount value of the deposit, including fee - * @param wire where should the funds be wired - * @param deadline what was the requested wire transfer deadline - * @param tiny did the exchange defer this transfer because it is too small? - * @param done did the exchange claim that it made a transfer? - */ -static void -wire_missing_cb (void *cls, - uint64_t rowid, - const struct TALER_CoinSpendPublicKeyP *coin_pub, - const struct TALER_Amount *amount, - const json_t *wire, - struct GNUNET_TIME_Absolute deadline, - /* bool? */ int tiny, - /* bool? */ int done) -{ - (void) cls; - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_amount_lag, - &total_amount_lag, - amount)); - if ( (GNUNET_YES == tiny) && - (0 > TALER_amount_cmp (amount, - &tiny_amount)) ) - return; /* acceptable, amount was tiny */ - report (report_lags, - json_pack ("{s:I, s:o, s:o, s:s, s:o, s:O}", - "row", (json_int_t) rowid, - "amount", TALER_JSON_from_amount (amount), - "deadline", json_from_time_abs (deadline), - "claimed_done", (done) ? "yes" : "no", - "coin_pub", GNUNET_JSON_from_data_auto (coin_pub), - "account", wire)); - -} - - -/** - * Checks that all wire transfers that should have happened - * (based on deposits) have indeed happened. - */ -static void -check_for_required_transfers () -{ - struct GNUNET_TIME_Absolute next_timestamp; - enum GNUNET_DB_QueryStatus qs; - - next_timestamp = GNUNET_TIME_absolute_get (); - (void) GNUNET_TIME_round_abs (&next_timestamp); - /* Subtract #GRACE_PERIOD, so we can be a bit behind in processing - without immediately raising undue concern */ - next_timestamp = GNUNET_TIME_absolute_subtract (next_timestamp, - GRACE_PERIOD); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing exchange's unfinished deposits (deadline: %s)\n", - GNUNET_STRINGS_absolute_time_to_string (next_timestamp)); - qs = edb->select_deposits_missing_wire (edb->cls, - esession, - pp.last_timestamp, - next_timestamp, - &wire_missing_cb, - &next_timestamp); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - pp.last_timestamp = next_timestamp; - /* conclude with success */ - commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); - GNUNET_SCHEDULER_shutdown (); -} - - -/* ***************************** Analyze reserves_out ************************ */ - -/** - * Clean up after processing wire out data. - */ -static void -conclude_wire_out () -{ - GNUNET_CONTAINER_multihashmap_destroy (out_map); - out_map = NULL; - check_for_required_transfers (); -} - - -/** - * Check that @a want is within #TIME_TOLERANCE of @a have. - * Otherwise report an inconsistency in row @a rowid of @a table. - * - * @param table where is the inconsistency (if any) - * @param rowid what is the row - * @param want what is the expected time - * @param have what is the time we got - */ -static void -check_time_difference (const char *table, - uint64_t rowid, - struct GNUNET_TIME_Absolute want, - struct GNUNET_TIME_Absolute have) -{ - struct GNUNET_TIME_Relative delta; - char *details; - - if (have.abs_value_us > want.abs_value_us) - delta = GNUNET_TIME_absolute_get_difference (want, - have); - else - delta = GNUNET_TIME_absolute_get_difference (have, - want); - if (delta.rel_value_us <= TIME_TOLERANCE.rel_value_us) - return; - - GNUNET_asprintf (&details, - "execution date mismatch (%s)", - GNUNET_STRINGS_relative_time_to_string (delta, - GNUNET_YES)); - report (report_row_minor_inconsistencies, - json_pack ("{s:s, s:I, s:s}", - "table", table, - "row", (json_int_t) rowid, - "diagnostic", details)); - GNUNET_free (details); -} - - -/** - * Function called with details about outgoing wire transfers - * as claimed by the exchange DB. - * - * @param cls a `struct WireAccount` - * @param rowid unique serial ID for the refresh session in our DB - * @param date timestamp of the transfer (roughly) - * @param wtid wire transfer subject - * @param wire wire transfer details of the receiver - * @param amount amount that was wired - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -wire_out_cb (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute date, - const struct TALER_WireTransferIdentifierRawP *wtid, - const json_t *wire, - const struct TALER_Amount *amount) -{ - struct WireAccount *wa = cls; - struct GNUNET_HashCode key; - struct ReserveOutInfo *roi; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Exchange wire OUT at %s of %s with WTID %s\n", - GNUNET_STRINGS_absolute_time_to_string (date), - TALER_amount2s (amount), - TALER_B2S (wtid)); - GNUNET_CRYPTO_hash (wtid, - sizeof (struct TALER_WireTransferIdentifierRawP), - &key); - roi = GNUNET_CONTAINER_multihashmap_get (out_map, - &key); - if (NULL == roi) - { - /* Wire transfer was not made (yet) at all (but would have been - justified), so the entire amount is missing / still to be done. - This is moderately harmless, it might just be that the aggreator - has not yet fully caught up with the transfers it should do. */ - report (report_wire_out_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", - "row", (json_int_t) rowid, - "amount_wired", TALER_JSON_from_amount (&zero), - "amount_justified", TALER_JSON_from_amount (amount), - "wtid", GNUNET_JSON_from_data_auto (wtid), - "timestamp", json_from_time_abs (date), - "diagnostic", "wire transfer not made (yet?)", - "account_section", wa->section_name)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_out_minus, - &total_bad_amount_out_minus, - amount)); - return GNUNET_OK; - } - { - char *payto_uri; - - payto_uri = TALER_JSON_wire_to_payto (wire); - if (0 != strcasecmp (payto_uri, - roi->details.credit_account_url)) - { - /* Destination bank account is wrong in actual wire transfer, so - we should count the wire transfer as entirely spurious, and - additionally consider the justified wire transfer as missing. */ - report (report_wire_out_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", - "row", (json_int_t) rowid, - "amount_wired", TALER_JSON_from_amount ( - &roi->details.amount), - "amount_justified", TALER_JSON_from_amount (&zero), - "wtid", GNUNET_JSON_from_data_auto (wtid), - "timestamp", json_from_time_abs (date), - "diagnostic", "recevier account mismatch", - "account_section", wa->section_name)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_out_plus, - &total_bad_amount_out_plus, - &roi->details.amount)); - report (report_wire_out_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", - "row", (json_int_t) rowid, - "amount_wired", TALER_JSON_from_amount (&zero), - "amount_justified", TALER_JSON_from_amount (amount), - "wtid", GNUNET_JSON_from_data_auto (wtid), - "timestamp", json_from_time_abs (date), - "diagnostic", "receiver account mismatch", - "account_section", wa->section_name)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_out_minus, - &total_bad_amount_out_minus, - amount)); - GNUNET_free (payto_uri); - goto cleanup; - } - GNUNET_free (payto_uri); - } - if (0 != TALER_amount_cmp (&roi->details.amount, - amount)) - { - report (report_wire_out_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", - "row", (json_int_t) rowid, - "amount_justified", TALER_JSON_from_amount (amount), - "amount_wired", TALER_JSON_from_amount ( - &roi->details.amount), - "wtid", GNUNET_JSON_from_data_auto (wtid), - "timestamp", json_from_time_abs (date), - "diagnostic", "wire amount does not match", - "account_section", wa->section_name)); - if (0 < TALER_amount_cmp (amount, - &roi->details.amount)) - { - /* amount > roi->details.amount: wire transfer was smaller than it should have been */ - struct TALER_Amount delta; - - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - amount, - &roi->details.amount)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_out_minus, - &total_bad_amount_out_minus, - &delta)); - } - else - { - /* roi->details.amount < amount: wire transfer was larger than it should have been */ - struct TALER_Amount delta; - - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - &roi->details.amount, - amount)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_out_plus, - &total_bad_amount_out_plus, - &delta)); - } - goto cleanup; - } - - check_time_difference ("wire_out", - rowid, - date, - roi->details.execution_date); -cleanup: - GNUNET_assert (GNUNET_OK == - free_roi (NULL, - &key, - roi)); - wa->pp.last_wire_out_serial_id = rowid + 1; - return GNUNET_OK; -} - - -/** - * Closure for #check_rc_matches - */ -struct CheckMatchContext -{ - - /** - * Reserve operation looking for a match - */ - const struct ReserveOutInfo *roi; - - /** - * Set to #GNUNET_YES if we found a match. - */ - int found; -}; - - -/** - * Check if any of the reserve closures match the given wire transfer. - * - * @param cls a `struct CheckMatchContext` - * @param key key of @a value in #reserve_closures - * @param value a `struct ReserveClosure` - */ -static int -check_rc_matches (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct CheckMatchContext *ctx = cls; - struct ReserveClosure *rc = value; - - if ( (0 == GNUNET_memcmp (&ctx->roi->details.wtid, - &rc->wtid)) && - (0 == strcasecmp (rc->receiver_account, - ctx->roi->details.credit_account_url)) && - (0 == TALER_amount_cmp (&rc->amount, - &ctx->roi->details.amount)) ) - { - check_time_difference ("reserves_closures", - rc->rowid, - rc->execution_date, - ctx->roi->details.execution_date); - ctx->found = GNUNET_YES; - free_rc (NULL, - key, - rc); - return GNUNET_NO; - } - return GNUNET_OK; -} - - -/** - * Check whether the given transfer was justified by a reserve closure. If - * not, complain that we failed to match an entry from #out_map. This means a - * wire transfer was made without proper justification. - * - * @param cls a `struct WireAccount` - * @param key unused key - * @param value the `struct ReserveOutInfo` to report - * @return #GNUNET_OK - */ -static int -complain_out_not_found (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct WireAccount *wa = cls; - struct ReserveOutInfo *roi = value; - struct GNUNET_HashCode rkey; - struct CheckMatchContext ctx = { - .roi = roi, - .found = GNUNET_NO - }; - - (void) key; - hash_rc (roi->details.credit_account_url, - &roi->details.wtid, - &rkey); - GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, - &rkey, - &check_rc_matches, - &ctx); - if (GNUNET_YES == ctx.found) - return GNUNET_OK; - report (report_wire_out_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", - "row", (json_int_t) 0, - "amount_wired", TALER_JSON_from_amount ( - &roi->details.amount), - "amount_justified", TALER_JSON_from_amount (&zero), - "wtid", GNUNET_JSON_from_data_auto (&roi->details.wtid), - "timestamp", json_from_time_abs ( - roi->details.execution_date), - "account_section", - wa->section_name, - "diagnostic", - "justification for wire transfer not found")); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_out_plus, - &total_bad_amount_out_plus, - &roi->details.amount)); - return GNUNET_OK; -} - - -/** - * Main function for processing 'reserves_out' data. We start by going over - * the DEBIT transactions this time, and then verify that all of them are - * justified by 'reserves_out'. - * - * @param cls `struct WireAccount` with a wire account list to process - */ -static void -process_debits (void *cls); - - -/** - * Go over the "wire_out" table of the exchange and - * verify that all wire outs are in that table. - * - * @param wa wire account we are processing - */ -static void -check_exchange_wire_out (struct WireAccount *wa) -{ - enum GNUNET_DB_QueryStatus qs; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing exchange's wire OUT table for account `%s'\n", - wa->section_name); - qs = edb->select_wire_out_above_serial_id_by_account (edb->cls, - esession, - wa->section_name, - wa->pp. - last_wire_out_serial_id, - &wire_out_cb, - wa); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - GNUNET_CONTAINER_multihashmap_iterate (out_map, - &complain_out_not_found, - wa); - /* clean up */ - GNUNET_CONTAINER_multihashmap_iterate (out_map, - &free_roi, - NULL); - process_debits (wa->next); -} - - -/** - * This function is called for all transactions that - * are debited from the exchange's account (outgoing - * transactions). - * - * @param cls `struct WireAccount` with current wire account to process - * @param http_status_code http status of the request - * @param ec error code in case something went wrong - * @param row_off identification of the position at which we are querying - * @param details details about the wire transfer - * @param json original response in JSON format - * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration - */ -static int -history_debit_cb (void *cls, - unsigned int http_status_code, - enum TALER_ErrorCode ec, - uint64_t row_off, - const struct TALER_BANK_DebitDetails *details, - const json_t *json) -{ - struct WireAccount *wa = cls; - struct ReserveOutInfo *roi; - size_t slen; - - (void) json; - if (NULL == details) - { - wa->dhh = NULL; - if (TALER_EC_NONE != ec) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Error fetching debit history of account %s: %u/%u!\n", - wa->section_name, - http_status_code, - (unsigned int) ec); - commit (GNUNET_DB_STATUS_HARD_ERROR); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return GNUNET_SYSERR; - } - check_exchange_wire_out (wa); - return GNUNET_OK; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing bank DEBIT at %s of %s with WTID %s\n", - GNUNET_STRINGS_absolute_time_to_string (details->execution_date), - TALER_amount2s (&details->amount), - TALER_B2S (&details->wtid)); - /* Update offset */ - wa->out_wire_off = row_off; - slen = strlen (details->credit_account_url) + 1; - roi = GNUNET_malloc (sizeof (struct ReserveOutInfo) - + slen); - GNUNET_CRYPTO_hash (&details->wtid, - sizeof (details->wtid), - &roi->subject_hash); - roi->details.amount = details->amount; - roi->details.execution_date = details->execution_date; - roi->details.wtid = details->wtid; - roi->details.credit_account_url = (const char *) &roi[1]; - memcpy (&roi[1], - details->credit_account_url, - slen); - if (GNUNET_OK != - GNUNET_CONTAINER_multihashmap_put (out_map, - &roi->subject_hash, - roi, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) - { - char *diagnostic; - - GNUNET_asprintf (&diagnostic, - "duplicate subject hash `%s'", - TALER_B2S (&roi->subject_hash)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_wire_format_amount, - &total_wire_format_amount, - &details->amount)); - report (report_wire_format_inconsistencies, - json_pack ("{s:o, s:I, s:s}", - "amount", TALER_JSON_from_amount (&details->amount), - "wire_offset", (json_int_t) row_off, - "diagnostic", diagnostic)); - GNUNET_free (diagnostic); - return GNUNET_OK; - } - return GNUNET_OK; -} - - -/** - * Main function for processing 'reserves_out' data. We start by going over - * the DEBIT transactions this time, and then verify that all of them are - * justified by 'reserves_out'. - * - * @param cls `struct WireAccount` with a wire account list to process - */ -static void -process_debits (void *cls) -{ - struct WireAccount *wa = cls; - - /* skip accounts where DEBIT is not enabled */ - while ( (NULL != wa) && - (GNUNET_NO == wa->watch_debit) ) - wa = wa->next; - if (NULL == wa) - { - /* end of iteration, now check wire_out to see - if it matches #out_map */ - conclude_wire_out (); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Checking bank DEBIT records of account `%s'\n", - wa->section_name); - GNUNET_assert (NULL == wa->dhh); - wa->dhh = TALER_BANK_debit_history (ctx, - &wa->auth, - wa->out_wire_off, - INT64_MAX, - &history_debit_cb, - wa); - if (NULL == wa->dhh) - { - fprintf (stderr, - "Failed to obtain bank transaction history for `%s'\n", - wa->section_name); - commit (GNUNET_DB_STATUS_HARD_ERROR); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } -} - - -/** - * Begin analyzing wire_out. - */ -static void -begin_debit_audit () -{ - out_map = GNUNET_CONTAINER_multihashmap_create (1024, - GNUNET_YES); - process_debits (wa_head); -} - - -/* ***************************** Analyze reserves_in ************************ */ - -/** - * Conclude the credit history check by logging entries that - * were not found and freeing resources. Then move on to - * processing debits. - */ -static void -conclude_credit_history (void) -{ - GNUNET_CONTAINER_multihashmap_destroy (in_map); - in_map = NULL; - /* credit done, now check debits */ - begin_debit_audit (); -} - - -/** - * Function called with details about incoming wire transfers - * as claimed by the exchange DB. - * - * @param cls a `struct WireAccount` we are processing - * @param rowid unique serial ID for the entry in our DB - * @param reserve_pub public key of the reserve (also the WTID) - * @param credit amount that was received - * @param sender_account_details payto://-URL of the sender's bank account - * @param wire_reference unique identifier for the wire transfer - * @param execution_date when did we receive the funds - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -reserve_in_cb (void *cls, - uint64_t rowid, - const struct - TALER_ReservePublicKeyP *reserve_pub, - const struct TALER_Amount *credit, - const char *sender_account_details, - uint64_t wire_reference, - struct GNUNET_TIME_Absolute - execution_date) -{ - struct WireAccount *wa = cls; - struct ReserveInInfo *rii; - size_t slen; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing exchange wire IN (%llu) at %s of %s with reserve_pub %s\n", - (unsigned long long) rowid, - GNUNET_STRINGS_absolute_time_to_string (execution_date), - TALER_amount2s (credit), - TALER_B2S (reserve_pub)); - slen = strlen (sender_account_details) + 1; - rii = GNUNET_malloc (sizeof (struct ReserveInInfo) - + slen); - rii->rowid = rowid; - rii->details.amount = *credit; - rii->details.execution_date = execution_date; - rii->details.reserve_pub = *reserve_pub; - rii->details.debit_account_url = (const char *) &rii[1]; - memcpy (&rii[1], - sender_account_details, - slen); - GNUNET_CRYPTO_hash (&wire_reference, - sizeof (uint64_t), - &rii->row_off_hash); - if (GNUNET_OK != - GNUNET_CONTAINER_multihashmap_put (in_map, - &rii->row_off_hash, - rii, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) - { - report (report_row_inconsistencies, - json_pack ("{s:s, s:I, s:o, s:s}", - "table", "reserves_in", - "row", (json_int_t) rowid, - "wire_offset_hash", GNUNET_JSON_from_data_auto ( - &rii->row_off_hash), - "diagnostic", "duplicate wire offset")); - GNUNET_free (rii); - return GNUNET_OK; - } - wa->pp.last_reserve_in_serial_id = rowid + 1; - return GNUNET_OK; -} - - -/** - * Complain that we failed to match an entry from #in_map. - * - * @param cls a `struct WireAccount` - * @param key unused key - * @param value the `struct ReserveInInfo` to free - * @return #GNUNET_OK - */ -static int -complain_in_not_found (void *cls, - const struct GNUNET_HashCode *key, - void *value) -{ - struct WireAccount *wa = cls; - struct ReserveInInfo *rii = value; - - (void) key; - report (report_reserve_in_inconsistencies, - json_pack ("{s:I, s:o, s:o, s:o, s:o, s:s, s:s}", - "row", (json_int_t) rii->rowid, - "amount_exchange_expected", TALER_JSON_from_amount ( - &rii->details.amount), - "amount_wired", TALER_JSON_from_amount (&zero), - "reserve_pub", GNUNET_JSON_from_data_auto ( - &rii->details.reserve_pub), - "timestamp", json_from_time_abs ( - rii->details.execution_date), - "account", wa->section_name, - "diagnostic", - "incoming wire transfer claimed by exchange not found")); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_in_minus, - &total_bad_amount_in_minus, - &rii->details.amount)); - return GNUNET_OK; -} - - -/** - * Start processing the next wire account. - * Shuts down if we are done. - * - * @param cls `struct WireAccount` with a wire account list to process - */ -static void -process_credits (void *cls); - - -/** - * This function is called for all transactions that - * are credited to the exchange's account (incoming - * transactions). - * - * @param cls `struct WireAccount` we are processing - * @param http_status HTTP status returned by the bank - * @param ec error code in case something went wrong - * @param row_off identification of the position at which we are querying - * @param details details about the wire transfer - * @param json raw response - * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration - */ -static int -history_credit_cb (void *cls, - unsigned int http_status, - enum TALER_ErrorCode ec, - uint64_t row_off, - const struct TALER_BANK_CreditDetails *details, - const json_t *json) -{ - struct WireAccount *wa = cls; - struct ReserveInInfo *rii; - struct GNUNET_HashCode key; - - (void) json; - if (NULL == details) - { - wa->chh = NULL; - if (TALER_EC_NONE != ec) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Error fetching credit history of account %s: %u/%u!\n", - wa->section_name, - http_status, - (unsigned int) ec); - commit (GNUNET_DB_STATUS_HARD_ERROR); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return GNUNET_SYSERR; - } - /* end of operation */ - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Reconciling CREDIT processing of account `%s'\n", - wa->section_name); - GNUNET_CONTAINER_multihashmap_iterate (in_map, - &complain_in_not_found, - wa); - /* clean up before 2nd phase */ - GNUNET_CONTAINER_multihashmap_iterate (in_map, - &free_rii, - NULL); - process_credits (wa->next); - return GNUNET_OK; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing bank CREDIT at %s of %s with Reserve-pub %s\n", - GNUNET_STRINGS_absolute_time_to_string (details->execution_date), - TALER_amount2s (&details->amount), - TALER_B2S (&details->reserve_pub)); - GNUNET_CRYPTO_hash (&row_off, - sizeof (row_off), - &key); - rii = GNUNET_CONTAINER_multihashmap_get (in_map, - &key); - if (NULL == rii) - { - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to find wire transfer at `%s' in exchange database. Audit ends at this point in time.\n", - GNUNET_STRINGS_absolute_time_to_string ( - details->execution_date)); - wa->chh = NULL; - process_credits (wa->next); - return GNUNET_SYSERR; /* not an error, just end of processing */ - } - - /* Update offset */ - wa->in_wire_off = row_off; - /* compare records with expected data */ - if (0 != GNUNET_memcmp (&details->reserve_pub, - &rii->details.reserve_pub)) - { - report (report_reserve_in_inconsistencies, - json_pack ("{s:I, s:I, s:o, s:o, s:o, s:o, s:s}", - "row", (json_int_t) rii->rowid, - "bank_row", (json_int_t) row_off, - "amount_exchange_expected", TALER_JSON_from_amount ( - &rii->details.amount), - "amount_wired", TALER_JSON_from_amount (&zero), - "reserve_pub", GNUNET_JSON_from_data_auto ( - &rii->details.reserve_pub), - "timestamp", json_from_time_abs ( - rii->details.execution_date), - "diagnostic", "wire subject does not match")); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_in_minus, - &total_bad_amount_in_minus, - &rii->details.amount)); - report (report_reserve_in_inconsistencies, - json_pack ("{s:I, s:I, s:o, s:o, s:o, s:o, s:s}", - "row", (json_int_t) rii->rowid, - "bank_row", (json_int_t) row_off, - "amount_exchange_expected", TALER_JSON_from_amount ( - &zero), - "amount_wired", TALER_JSON_from_amount ( - &details->amount), - "reserve_pub", GNUNET_JSON_from_data_auto ( - &details->reserve_pub), - "timestamp", json_from_time_abs ( - details->execution_date), - "diagnostic", "wire subject does not match")); - - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_in_plus, - &total_bad_amount_in_plus, - &details->amount)); - goto cleanup; - } - if (0 != TALER_amount_cmp (&rii->details.amount, - &details->amount)) - { - report (report_reserve_in_inconsistencies, - json_pack ("{s:I, s:I, s:o, s:o, s:o, s:o, s:s}", - "row", (json_int_t) rii->rowid, - "bank_row", (json_int_t) row_off, - "amount_exchange_expected", TALER_JSON_from_amount ( - &rii->details.amount), - "amount_wired", TALER_JSON_from_amount ( - &details->amount), - "reserve_pub", GNUNET_JSON_from_data_auto ( - &details->reserve_pub), - "timestamp", json_from_time_abs ( - details->execution_date), - "diagnostic", "wire amount does not match")); - if (0 < TALER_amount_cmp (&details->amount, - &rii->details.amount)) - { - /* details->amount > rii->details.amount: wire transfer was larger than it should have been */ - struct TALER_Amount delta; - - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - &details->amount, - &rii->details.amount)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_in_plus, - &total_bad_amount_in_plus, - &delta)); - } - else - { - /* rii->details.amount < details->amount: wire transfer was smaller than it should have been */ - struct TALER_Amount delta; - - GNUNET_break (GNUNET_OK == - TALER_amount_subtract (&delta, - &rii->details.amount, - &details->amount)); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_bad_amount_in_minus, - &total_bad_amount_in_minus, - &delta)); - } - goto cleanup; - } - if (0 != strcasecmp (details->debit_account_url, - rii->details.debit_account_url)) - { - report (report_missattribution_in_inconsistencies, - json_pack ("{s:o, s:I, s:I, s:o}", - "amount", TALER_JSON_from_amount (&rii->details.amount), - "row", (json_int_t) rii->rowid, - "bank_row", (json_int_t) row_off, - "reserve_pub", GNUNET_JSON_from_data_auto ( - &rii->details.reserve_pub))); - GNUNET_break (GNUNET_OK == - TALER_amount_add (&total_missattribution_in, - &total_missattribution_in, - &rii->details.amount)); - } - if (details->execution_date.abs_value_us != - rii->details.execution_date.abs_value_us) - { - report (report_row_minor_inconsistencies, - json_pack ("{s:s, s:I, s:I, s:s}", - "table", "reserves_in", - "row", (json_int_t) rii->rowid, - "bank_row", (json_int_t) row_off, - "diagnostic", "execution date mismatch")); - } -cleanup: - GNUNET_assert (GNUNET_OK == - free_rii (NULL, - &key, - rii)); - return GNUNET_OK; -} - - -/* ***************************** Setup logic ************************ */ - - -/** - * Start processing the next wire account. - * Shuts down if we are done. - * - * @param cls `struct WireAccount` with a wire account list to process - */ -static void -process_credits (void *cls) -{ - struct WireAccount *wa = cls; - enum GNUNET_DB_QueryStatus qs; - - /* skip accounts where CREDIT is not enabled */ - while ( (NULL != wa) && - (GNUNET_NO == wa->watch_credit) ) - wa = wa->next; - if (NULL == wa) - { - /* done with all accounts, conclude check */ - conclude_credit_history (); - return; - } - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Analyzing exchange's wire IN table for account `%s'\n", - wa->section_name); - qs = edb->select_reserves_in_above_serial_id_by_account (edb->cls, - esession, - wa->section_name, - wa->pp. - last_reserve_in_serial_id, - &reserve_in_cb, - wa); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Starting bank CREDIT history of account `%s'\n", - wa->section_name); - wa->chh = TALER_BANK_credit_history (ctx, - &wa->auth, - wa->in_wire_off, - INT64_MAX, - &history_credit_cb, - wa); - if (NULL == wa->chh) - { - fprintf (stderr, - "Failed to obtain bank transaction history\n"); - commit (GNUNET_DB_STATUS_HARD_ERROR); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } -} - - -/** - * Begin audit of CREDITs to the exchange. - */ -static void -begin_credit_audit () -{ - in_map = GNUNET_CONTAINER_multihashmap_create (1024, - GNUNET_YES); - /* now go over all bank accounts and check delta with in_map */ - process_credits (wa_head); -} - - -/** - * Function called about reserve closing operations - * the aggregator triggered. - * - * @param cls closure - * @param rowid row identifier used to uniquely identify the reserve closing operation - * @param execution_date when did we execute the close operation - * @param amount_with_fee how much did we debit the reserve - * @param closing_fee how much did we charge for closing the reserve - * @param reserve_pub public key of the reserve - * @param receiver_account where did we send the funds, in payto://-format - * @param wtid identifier used for the wire transfer - * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop - */ -static int -reserve_closed_cb (void *cls, - uint64_t rowid, - struct GNUNET_TIME_Absolute execution_date, - const struct TALER_Amount *amount_with_fee, - const struct TALER_Amount *closing_fee, - const struct TALER_ReservePublicKeyP *reserve_pub, - const char *receiver_account, - const struct TALER_WireTransferIdentifierRawP *wtid) -{ - struct ReserveClosure *rc; - struct GNUNET_HashCode key; - - (void) cls; - rc = GNUNET_new (struct ReserveClosure); - if (GNUNET_SYSERR == - TALER_amount_subtract (&rc->amount, - amount_with_fee, - closing_fee)) - { - report (report_row_inconsistencies, - json_pack ("{s:s, s:I, s:o, s:o, s:o, s:s}", - "table", "reserves_closures", - "row", (json_int_t) rowid, - "reserve_pub", GNUNET_JSON_from_data_auto (reserve_pub), - "amount_with_fee", TALER_JSON_from_amount ( - amount_with_fee), - "closing_fee", TALER_JSON_from_amount (closing_fee), - "diagnostic", "closing fee above total amount")); - GNUNET_free (rc); - return GNUNET_OK; - } - pp.last_reserve_close_uuid - = GNUNET_MAX (pp.last_reserve_close_uuid, - rowid + 1); - rc->receiver_account = GNUNET_strdup (receiver_account); - rc->wtid = *wtid; - rc->execution_date = execution_date; - rc->rowid = rowid; - hash_rc (receiver_account, - wtid, - &key); - (void) GNUNET_CONTAINER_multihashmap_put (reserve_closures, - &key, - rc, - GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); - return GNUNET_OK; -} - - -/** - * Start the database transactions and begin the audit. - */ -static void -begin_transaction () -{ - int ret; - - ret = adb->start (adb->cls, - asession); - if (GNUNET_OK != ret) - { - GNUNET_break (0); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - edb->preflight (edb->cls, - esession); - ret = edb->start (edb->cls, - esession, - "wire auditor"); - if (GNUNET_OK != ret) - { - GNUNET_break (0); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - for (struct WireAccount *wa = wa_head; - NULL != wa; - wa = wa->next) - { - wa->qsx = adb->get_wire_auditor_account_progress (adb->cls, - asession, - &master_pub, - wa->section_name, - &wa->pp, - &wa->in_wire_off, - &wa->out_wire_off); - if (0 > wa->qsx) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == wa->qsx); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - wa->start_pp = wa->pp; - } - qsx_gwap = adb->get_wire_auditor_progress (adb->cls, - asession, - &master_pub, - &pp); - if (0 > qsx_gwap) - { - GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qsx_gwap); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qsx_gwap) - { - GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, - _ ( - "First analysis using this auditor, starting audit from scratch\n")); - } - else - { - start_pp = pp; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Resuming audit at %s / %llu\n", - GNUNET_STRINGS_absolute_time_to_string (pp.last_timestamp), - (unsigned long long) pp.last_reserve_close_uuid); - } - - { - enum GNUNET_DB_QueryStatus qs; - - qs = edb->select_reserve_closed_above_serial_id (edb->cls, - esession, - pp. - last_reserve_close_uuid, - &reserve_closed_cb, - NULL); - if (0 > qs) - { - GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - } - begin_credit_audit (); -} - - -/** - * Function called with information about a wire account. Adds the - * account to our list for processing (if it is enabled and we can - * load the plugin). - * - * @param cls closure, NULL - * @param ai account information - */ -static void -process_account_cb (void *cls, - const struct TALER_EXCHANGEDB_AccountInfo *ai) -{ - struct WireAccount *wa; - - (void) cls; - if ( (GNUNET_NO == ai->debit_enabled) && - (GNUNET_NO == ai->credit_enabled) ) - return; /* not an active exchange account */ - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Found exchange account `%s'\n", - ai->section_name); - wa = GNUNET_new (struct WireAccount); - wa->section_name = GNUNET_strdup (ai->section_name); - wa->watch_debit = ai->debit_enabled; - wa->watch_credit = ai->credit_enabled; - if (GNUNET_OK != - TALER_BANK_auth_parse_cfg (cfg, - ai->section_name, - &wa->auth)) - { - GNUNET_break (0); - GNUNET_free (wa->section_name); - GNUNET_free (wa); - fprintf (stderr, - "Failed to access bank account `%s'\n", - wa->section_name); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - GNUNET_CONTAINER_DLL_insert (wa_head, - wa_tail, - wa); -} - - -/** - * Main function that will be run. - * - * @param cls closure - * @param args remaining command-line arguments - * @param cfgfile name of the configuration file used (for saving, can be NULL!) - * @param c configuration - */ -static void -run (void *cls, - char *const *args, - const char *cfgfile, - const struct GNUNET_CONFIGURATION_Handle *c) -{ - static const struct TALER_MasterPublicKeyP zeromp; - - (void) cls; - (void) args; - (void) cfgfile; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Launching auditor\n"); - start_time = GNUNET_TIME_absolute_get (); - cfg = c; - if (GNUNET_OK != - TALER_config_get_amount (cfg, - "auditor", - "TINY_AMOUNT", - &tiny_amount)) - { - GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, - "auditor", - "TINY_AMOUNT", - "invalid amount"); - global_ret = 1; - return; - } - if (0 == GNUNET_memcmp (&zeromp, - &master_pub)) - { - /* -m option not given, try configuration */ - char *master_public_key_str; - - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "exchange", - "MASTER_PUBLIC_KEY", - &master_public_key_str)) - { - fprintf (stderr, - "Pass option -m or set it in the configuration!\n"); - GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, - "exchange", - "MASTER_PUBLIC_KEY"); - global_ret = 1; - return; - } - if (GNUNET_OK != - GNUNET_CRYPTO_eddsa_public_key_from_string (master_public_key_str, - strlen ( - master_public_key_str), - &master_pub.eddsa_pub)) - { - fprintf (stderr, - "Invalid master public key given in configuration file."); - GNUNET_free (master_public_key_str); - global_ret = 1; - return; - } - GNUNET_free (master_public_key_str); - } /* end of -m not given */ - - if (GNUNET_OK != - TALER_config_get_currency (cfg, - ¤cy)) - { - global_ret = 1; - return; - } - if (NULL == - (edb = TALER_EXCHANGEDB_plugin_load (cfg))) - { - fprintf (stderr, - "Failed to initialize exchange database plugin.\n"); - global_ret = 1; - return; - } - if (NULL == - (adb = TALER_AUDITORDB_plugin_load (cfg))) - { - fprintf (stderr, - "Failed to initialize auditor database plugin.\n"); - global_ret = 1; - TALER_EXCHANGEDB_plugin_unload (edb); - return; - } - if (restart) - { - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Full audit restart requested, dropping old audit data.\n"); - GNUNET_break (GNUNET_OK == - adb->drop_tables (adb->cls, - GNUNET_NO)); - TALER_AUDITORDB_plugin_unload (adb); - if (NULL == - (adb = TALER_AUDITORDB_plugin_load (cfg))) - { - fprintf (stderr, - "Failed to initialize auditor database plugin after drop.\n"); - global_ret = 1; - TALER_EXCHANGEDB_plugin_unload (edb); - return; - } - GNUNET_break (GNUNET_OK == - adb->create_tables (adb->cls)); - } - GNUNET_SCHEDULER_add_shutdown (&do_shutdown, - NULL); - ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, - &rc); - rc = GNUNET_CURL_gnunet_rc_create (ctx); - if (NULL == ctx) - { - GNUNET_break (0); - return; - } - esession = edb->get_session (edb->cls); - if (NULL == esession) - { - fprintf (stderr, - "Failed to initialize exchange session.\n"); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - asession = adb->get_session (adb->cls); - if (NULL == asession) - { - fprintf (stderr, - "Failed to initialize auditor session.\n"); - global_ret = 1; - GNUNET_SCHEDULER_shutdown (); - return; - } - reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, - GNUNET_NO); - GNUNET_assert (NULL != - (report_wire_out_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_reserve_in_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_row_minor_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_wire_format_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_row_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_missattribution_in_inconsistencies = json_array ())); - GNUNET_assert (NULL != - (report_lags = json_array ())); - GNUNET_assert (NULL != - (report_closure_lags = json_array ())); - GNUNET_assert (NULL != - (report_account_progress = json_array ())); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_amount_out_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_amount_out_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_amount_in_plus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_bad_amount_in_minus)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_missattribution_in)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_amount_lag)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_closure_amount_lag)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &total_wire_format_amount)); - GNUNET_assert (GNUNET_OK == - TALER_amount_get_zero (currency, - &zero)); - TALER_EXCHANGEDB_find_accounts (cfg, - &process_account_cb, - NULL); - begin_transaction (); -} - - -/** - * The main function of the database initialization tool. - * Used to initialize the Taler Exchange's database. - * - * @param argc number of arguments from the command line - * @param argv command line arguments - * @return 0 ok, 1 on error - */ -int -main (int argc, - char *const *argv) -{ - const struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_option_base32_auto ('m', - "exchange-key", - "KEY", - "public key of the exchange (Crockford base32 encoded)", - &master_pub), - GNUNET_GETOPT_option_flag ('r', - "restart", - "restart audit from the beginning (required on first run)", - &restart), - GNUNET_GETOPT_option_timetravel ('T', - "timetravel"), - GNUNET_GETOPT_OPTION_END - }; - - /* force linker to link against libtalerutil; if we do - not do this, the linker may "optimize" libtalerutil - away and skip #TALER_OS_init(), which we do need */ - (void) TALER_project_data_default (); - GNUNET_assert (GNUNET_OK == - GNUNET_log_setup ("taler-wire-auditor", - "MESSAGE", - NULL)); - if (GNUNET_OK != - GNUNET_PROGRAM_run (argc, - argv, - "taler-wire-auditor", - "Audit exchange database for consistency with the bank's wire transfers", - options, - &run, - NULL)) - return 1; - return global_ret; -} - - -/* end of taler-wire-auditor.c */ diff --git a/src/auditor/test-auditor.sh b/src/auditor/test-auditor.sh index 76d74d82c..236a11519 100755 --- a/src/auditor/test-auditor.sh +++ b/src/auditor/test-auditor.sh @@ -84,10 +84,10 @@ function audit_only () { # Also do incremental run $VALGRIND taler-auditor -L DEBUG -c $CONF -m $MASTER_PUB > test-audit-inc.json 2> test-audit-inc.log || exit_fail "auditor failed" echo -n "." - $VALGRIND taler-wire-auditor -L DEBUG -r -c $CONF -m $MASTER_PUB > test-wire-audit.json 2> test-wire-audit.log || exit_fail "wire auditor failed" + $VALGRIND taler-helper-auditor-wire -L DEBUG -r -c $CONF -m $MASTER_PUB > test-wire-audit.json 2> test-wire-audit.log || exit_fail "wire auditor failed" # Also do incremental run echo -n "." - $VALGRIND taler-wire-auditor -L DEBUG -c $CONF -m $MASTER_PUB > test-wire-audit-inc.json 2> test-wire-audit-inc.log || exit_fail "wire auditor failed" + $VALGRIND taler-helper-auditor-wire -L DEBUG -c $CONF -m $MASTER_PUB > test-wire-audit-inc.json 2> test-wire-audit-inc.log || exit_fail "wire auditor failed" echo " DONE" } @@ -255,7 +255,7 @@ echo -n "Check for lag detection... " # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then jq -e .lag_details[0] < test-wire-audit.json > /dev/null || exit_fail "Lag not detected in run without aggregator at age $DELTA" @@ -893,7 +893,7 @@ echo "===========15: deposit wire hash wrong=================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -930,7 +930,7 @@ echo "===========16: incorrect wire_out amount=================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -1023,7 +1023,7 @@ echo "===========17: incorrect wire_out timestamp=================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -1115,7 +1115,7 @@ echo "===========19: reserve closure done properly =================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -1193,7 +1193,7 @@ echo "===========21: reserve closure missreported =================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -1279,7 +1279,7 @@ echo "===========23: wire out calculations =================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -1400,7 +1400,7 @@ echo "=========25: inconsistent coin history=========" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then @@ -1493,7 +1493,7 @@ echo "===========27: duplicate WTID detection =================" # NOTE: This test is EXPECTED to fail for ~1h after # re-generating the test database as we do not # report lag of less than 1h (see GRACE_PERIOD in -# taler-wire-auditor.c) +# taler-helper-auditor-wire.c) if [ $DATABASE_AGE -gt 3600 ] then -- cgit v1.2.3