commit ce9de215f728e7734aed98b30c9d41629984848a
parent a1efc8e378c36c5b0585c5423b52979200631628
Author: Christian Grothoff <christian@grothoff.org>
Date: Thu, 29 Aug 2024 06:43:29 +0200
fix auditor inconsistency detection lag
Diffstat:
11 files changed, 173 insertions(+), 24 deletions(-)
diff --git a/src/auditor/taler-helper-auditor-transfer.c b/src/auditor/taler-helper-auditor-transfer.c
@@ -49,6 +49,11 @@ static TALER_ARL_DEF_PP (wire_batch_deposit_id);
static TALER_ARL_DEF_PP (wire_aggregation_id);
/**
+ * Total amount which the exchange did not transfer in time.
+ */
+static TALER_ARL_DEF_AB (total_amount_lag);
+
+/**
* Should we run checks that only work for exchange-internal audits?
*/
static int internal_checks;
@@ -137,6 +142,9 @@ import_wire_missing_cb (
GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
wc->err = qs;
}
+ TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_amount_lag),
+ &TALER_ARL_USE_AB (total_amount_lag),
+ total_amount);
}
@@ -196,12 +204,14 @@ struct AggregationContext
* a (batch) deposit.
*
* @param cls closure
+ * @param amount affected amount
* @param tracking_serial_id where in the table are we
* @param batch_deposit_serial_id which batch deposit was aggregated
*/
static void
clear_finished_transfer_cb (
void *cls,
+ const struct TALER_Amount *amount,
uint64_t tracking_serial_id,
uint64_t batch_deposit_serial_id)
{
@@ -220,12 +230,17 @@ clear_finished_transfer_cb (
/* Aggregated something twice or other error, report! */
GNUNET_break (0);
// FIXME: report more nicely!
+ return;
}
if (0 > qs)
{
GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs);
ac->err = qs;
+ return;
}
+ TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_amount_lag),
+ &TALER_ARL_USE_AB (total_amount_lag),
+ amount);
}
@@ -309,6 +324,13 @@ begin_transaction (void)
NULL);
if (0 > qs)
goto handle_db_error;
+
+ qs = TALER_ARL_adb->get_balance (
+ TALER_ARL_adb->cls,
+ TALER_ARL_GET_AB (total_amount_lag),
+ NULL);
+ if (0 > qs)
+ goto handle_db_error;
if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs)
{
GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE,
@@ -328,6 +350,7 @@ begin_transaction (void)
qs = check_for_completed_transfers ();
if (0 > qs)
goto handle_db_error;
+
qs = TALER_ARL_adb->update_auditor_progress (
TALER_ARL_adb->cls,
TALER_ARL_SET_PP (wire_batch_deposit_id),
@@ -342,6 +365,18 @@ begin_transaction (void)
NULL);
if (0 > qs)
goto handle_db_error;
+ qs = TALER_ARL_adb->update_balance (
+ TALER_ARL_adb->cls,
+ TALER_ARL_SET_AB (total_amount_lag),
+ NULL);
+ if (0 > qs)
+ goto handle_db_error;
+ qs = TALER_ARL_adb->insert_balance (
+ TALER_ARL_adb->cls,
+ TALER_ARL_SET_AB (total_amount_lag),
+ NULL);
+ if (0 > qs)
+ goto handle_db_error;
GNUNET_log (GNUNET_ERROR_TYPE_INFO,
"Concluded audit step at %llu/%llu\n",
(unsigned long long) TALER_ARL_USE_PP (wire_aggregation_id),
diff --git a/src/auditor/taler-helper-auditor-wire-debit.c b/src/auditor/taler-helper-auditor-wire-debit.c
@@ -230,11 +230,6 @@ static TALER_ARL_DEF_AB (total_bad_amount_out_plus);
static TALER_ARL_DEF_AB (total_bad_amount_out_minus);
/**
- * Total amount which the exchange did not transfer in time.
- */
-static TALER_ARL_DEF_AB (total_amount_lag);
-
-/**
* Total amount of reserve closures which the exchange did not transfer in time.
*/
static TALER_ARL_DEF_AB (total_closure_amount_lag);
@@ -535,7 +530,6 @@ commit (enum GNUNET_DB_QueryStatus qs)
TALER_ARL_SET_AB (total_wire_out),
TALER_ARL_SET_AB (total_bad_amount_out_plus),
TALER_ARL_SET_AB (total_bad_amount_out_minus),
- TALER_ARL_SET_AB (total_amount_lag),
TALER_ARL_SET_AB (total_closure_amount_lag),
TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total),
TALER_ARL_SET_AB (total_wire_out),
@@ -548,7 +542,6 @@ commit (enum GNUNET_DB_QueryStatus qs)
TALER_ARL_SET_AB (total_wire_out),
TALER_ARL_SET_AB (total_bad_amount_out_plus),
TALER_ARL_SET_AB (total_bad_amount_out_minus),
- TALER_ARL_SET_AB (total_amount_lag),
TALER_ARL_SET_AB (total_closure_amount_lag),
TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total),
TALER_ARL_SET_AB (total_wire_out),
@@ -1603,7 +1596,6 @@ begin_transaction (void)
TALER_ARL_GET_AB (total_wire_out),
TALER_ARL_GET_AB (total_bad_amount_out_plus),
TALER_ARL_GET_AB (total_bad_amount_out_minus),
- TALER_ARL_GET_AB (total_amount_lag),
TALER_ARL_GET_AB (total_closure_amount_lag),
TALER_ARL_GET_AB (wire_debit_duplicate_transfer_subject_total),
TALER_ARL_GET_AB (total_wire_out),
diff --git a/src/exchangedb/.gitignore b/src/exchangedb/.gitignore
@@ -15,3 +15,4 @@ test-exchangedb-populate-ready-deposit-postgres
test-exchangedb-populate-select-refunds-by-coin-postgres
exchange-0004.sql
exchange-0005.sql
+exchange-0006.sql
diff --git a/src/exchangedb/0006-dummy.sql b/src/exchangedb/0006-dummy.sql
diff --git a/src/exchangedb/Makefile.am b/src/exchangedb/Makefile.am
@@ -21,10 +21,12 @@ sqlinputs = \
0003-*.sql \
0004-*.sql \
0005-*.sql \
+ 0006-*.sql \
exchange-0002.sql.in \
exchange-0003.sql.in \
exchange-0004.sql.in \
- exchange-0005.sql.in
+ exchange-0005.sql.in \
+ exchange-0006.sql.in
sql_DATA = \
benchmark-0001.sql \
@@ -35,6 +37,7 @@ sql_DATA = \
exchange-0003.sql \
exchange-0004.sql \
exchange-0005.sql \
+ exchange-0006.sql \
drop.sql \
procedures.sql
@@ -49,6 +52,7 @@ CLEANFILES = \
exchange-0003.sql \
exchange-0004.sql \
exchange-0005.sql \
+ exchange-0006.sql \
procedures.sql
procedures.sql: procedures.sql.in exchange_do_*.sql
@@ -76,6 +80,11 @@ exchange-0005.sql: exchange-0005.sql.in 0005-*.sql
gcc -E -P -undef - < exchange-0005.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
chmod ugo-w $@
+exchange-0006.sql: exchange-0006.sql.in 0006-*.sql
+ chmod +w $@ 2> /dev/null || true
+ gcc -E -P -undef - < exchange-0006.sql.in 2>/dev/null | sed -e "s/--.*//" | awk 'NF' - >$@
+ chmod ugo-w $@
+
check_SCRIPTS = \
test_idempotency.sh
diff --git a/src/exchangedb/exchange-0006.sql.in b/src/exchangedb/exchange-0006.sql.in
@@ -0,0 +1,32 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2014--2023 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+--
+
+BEGIN;
+
+SELECT _v.register_patch('exchange-0006', NULL, NULL);
+SET search_path TO exchange;
+
+CREATE TYPE exchange_do_select_aggregations_above_serial_return_type
+ AS
+ (
+ batch_deposit_serial_id INT8,
+ aggregation_serial_id INT8,
+ total_amount taler_amount
+ );
+COMMENT ON TYPE exchange_do_select_aggregations_above_serial_return_type
+ IS 'Return type for exchange_do_select_aggregations_above_serial';
+
+COMMIT;
diff --git a/src/exchangedb/exchange_do_select_aggregations_above_serial.sql b/src/exchangedb/exchange_do_select_aggregations_above_serial.sql
@@ -0,0 +1,72 @@
+--
+-- This file is part of TALER
+-- Copyright (C) 2023 Taler Systems SA
+--
+-- TALER is free software; you can redistribute it and/or modify it under the
+-- terms of the GNU General Public License as published by the Free Software
+-- Foundation; either version 3, or (at your option) any later version.
+--
+-- TALER is distributed in the hope that it will be useful, but WITHOUT ANY
+-- WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+-- A PARTICULAR PURPOSE. See the GNU General Public License for more details.
+--
+-- You should have received a copy of the GNU General Public License along with
+-- TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/>
+--
+-- @author: Christian Grothoff
+
+CREATE OR REPLACE FUNCTION exchange_do_select_aggregations_above_serial(
+ IN in_min_serial_id INT8)
+RETURNS SETOF exchange_do_select_aggregations_above_serial_return_type
+LANGUAGE plpgsql
+AS $$
+DECLARE
+ aggregation CURSOR
+ FOR
+ SELECT
+ batch_deposit_serial_id
+ ,aggregation_serial_id
+ FROM aggregation_tracking
+ WHERE aggregation_serial_id >= in_min_serial_id
+ ORDER BY aggregation_serial_id ASC;
+DECLARE
+ my_total_val INT8; -- all deposits without wire
+DECLARE
+ my_total_frac INT8; -- all deposits without wire (fraction, not normalized)
+DECLARE
+ my_total taler_amount; -- amount that was originally deposited
+DECLARE
+ my_batch_record RECORD;
+DECLARE
+ i RECORD;
+BEGIN
+
+OPEN aggregation;
+LOOP
+ FETCH NEXT FROM aggregation INTO i;
+ EXIT WHEN NOT FOUND;
+
+ SELECT
+ SUM((cdep.amount_with_fee).val) AS total_val
+ ,SUM((cdep.amount_with_fee).frac::INT8) AS total_frac
+ INTO
+ my_batch_record
+ FROM coin_deposits cdep
+ WHERE cdep.batch_deposit_serial_id = i.batch_deposit_serial_id;
+
+ my_total_val=my_batch_record.total_val;
+ my_total_frac=my_batch_record.total_frac;
+
+ -- Normalize total amount
+ my_total.val = my_total_val + my_total_frac / 100000000;
+ my_total.frac = my_total_frac % 100000000;
+ RETURN NEXT (
+ i.batch_deposit_serial_id
+ ,i.aggregation_serial_id
+ ,my_total
+ );
+
+END LOOP;
+CLOSE aggregation;
+RETURN;
+END $$;
diff --git a/src/exchangedb/pg_helper.h b/src/exchangedb/pg_helper.h
@@ -100,27 +100,27 @@ struct PostgresClosure
* @param name name to prepare the statement under
* @param sql actual SQL text
*/
-#define PREPARE(pg,name,sql) \
+#define PREPARE(pg,name,sql) \
do { \
static struct { \
unsigned long long cnt; \
struct PostgresClosure *pg; \
- } preps[2]; /* 2 ctrs for taler-auditor-sync*/ \
- unsigned int off = 0; \
- \
- while ( (NULL != preps[off].pg) && \
- (pg != preps[off].pg) && \
- (off < sizeof(preps) / sizeof(*preps)) ) \
- off++; \
- GNUNET_assert (off < \
- sizeof(preps) / sizeof(*preps)); \
- if (preps[off].cnt < pg->prep_gen) \
+ } preps_[2]; /* 2 ctrs for taler-auditor-sync*/ \
+ unsigned int off_ = 0; \
+ \
+ while ( (NULL != preps_[off_].pg) && \
+ (pg != preps_[off_].pg) && \
+ (off_ < sizeof(preps_) / sizeof(*preps_)) ) \
+ off_++; \
+ GNUNET_assert (off_ < \
+ sizeof(preps_) / sizeof(*preps_)); \
+ if (preps_[off_].cnt < pg->prep_gen) \
{ \
struct GNUNET_PQ_PreparedStatement ps[] = { \
GNUNET_PQ_make_prepare (name, sql), \
GNUNET_PQ_PREPARED_STATEMENT_END \
}; \
- \
+ \
if (GNUNET_OK != \
GNUNET_PQ_prepare_statements (pg->conn, \
ps)) \
@@ -128,8 +128,8 @@ struct PostgresClosure
GNUNET_break (0); \
return GNUNET_DB_STATUS_HARD_ERROR; \
} \
- preps[off].pg = pg; \
- preps[off].cnt = pg->prep_gen; \
+ preps_[off_].pg = pg; \
+ preps_[off_].cnt = pg->prep_gen; \
} \
} while (0)
diff --git a/src/exchangedb/pg_select_aggregations_above_serial.c b/src/exchangedb/pg_select_aggregations_above_serial.c
@@ -1,6 +1,6 @@
/*
This file is part of TALER
- Copyright (C) 2023 Taler Systems SA
+ Copyright (C) 2023, 2024 Taler Systems SA
TALER is free software; you can redistribute it and/or modify it under the
terms of the GNU General Public License as published by the Free Software
@@ -67,12 +67,16 @@ aggregation_serial_helper_cb (void *cls,
unsigned int num_results)
{
struct AggregationSerialContext *dsc = cls;
+ struct PostgresClosure *pg = dsc->pg;
for (unsigned int i = 0; i<num_results; i++)
{
uint64_t tracking_rowid;
uint64_t batch_deposit_serial_id;
+ struct TALER_Amount amount;
struct GNUNET_PQ_ResultSpec rs[] = {
+ TALER_PQ_RESULT_SPEC_AMOUNT ("amount",
+ &amount),
GNUNET_PQ_result_spec_uint64 ("aggregation_serial_id",
&tracking_rowid),
GNUNET_PQ_result_spec_uint64 ("batch_deposit_serial_id",
@@ -90,6 +94,7 @@ aggregation_serial_helper_cb (void *cls,
return;
}
dsc->cb (dsc->cb_cls,
+ &amount,
tracking_rowid,
batch_deposit_serial_id);
GNUNET_PQ_cleanup_result (rs);
diff --git a/src/exchangedb/procedures.sql.in b/src/exchangedb/procedures.sql.in
@@ -53,5 +53,6 @@ SET search_path TO exchange;
#include "exchange_do_trigger_kyc_rule_for_account.sql"
#include "exchange_do_lookup_kyc_requirement_by_row.sql"
#include "exchange_do_insert_programmatic_legitimization_outcome.sql"
+#include "exchange_do_select_aggregations_above_serial.sql"
COMMIT;
diff --git a/src/include/taler_exchangedb_plugin.h b/src/include/taler_exchangedb_plugin.h
@@ -3526,12 +3526,14 @@ typedef void
* a (batch) deposit.
*
* @param cls closure
+ * @param amount affected amount
* @param tracking_serial_id where in the table are we
* @param batch_deposit_serial_id which batch deposit was aggregated
*/
typedef void
(*TALER_EXCHANGEDB_AggregationCallback)(
void *cls,
+ const struct TALER_Amount *amount,
uint64_t tracking_serial_id,
uint64_t batch_deposit_serial_id);
