taler-helper-auditor-wire-debit.c (58996B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2017-2024 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file auditor/taler-helper-auditor-wire-debit.c 18 * @brief audits that wire outgoing transfers match those from an exchange 19 * database. 20 * @author Christian Grothoff 21 * @author Özgür Kesim 22 * 23 * - We check that the outgoing wire transfers match those 24 * given in the 'wire_out' and 'reserve_closures' tables; 25 * any outgoing transfer MUST have a prior justification, 26 * so if one is missing we flag it (and never remove it). 27 * - We check that all wire transfers that should 28 * have been made, were actually made. If any were not made, 29 * we flag those, but may remove those flags if we later 30 * find that the wire transfers were made (wire transfers 31 * could be delayed due to AML/KYC or core-banking issues). 32 */ 33 #include "platform.h" 34 #include <gnunet/gnunet_util_lib.h> 35 #include <gnunet/gnunet_curl_lib.h> 36 #include "auditordb_lib.h" 37 /* WIRE_TRANSFER_OUT's callback typedef also lives in exchangedb_lib.h, so its 38 closure override must be established before that header is included. */ 39 struct WireAccount; 40 #define TALER_EXCHANGEDB_WIRE_TRANSFER_OUT_RESULT_CLOSURE struct WireAccount 41 #include "exchangedb_lib.h" 42 #include "taler/taler_json_lib.h" 43 #include "taler/taler_bank_service.h" 44 #include "taler/taler_signatures.h" 45 #include "report-lib.h" 46 #include "taler/taler_dbevents.h" 47 #include "auditor-database/delete_auditor_closure_lag.h" 48 #include "auditor-database/delete_wire_out_inconsistency_if_matching.h" 49 #include "auditor-database/event_listen.h" 50 #include "auditor-database/get_auditor_progress.h" 51 #include "auditor-database/get_balance.h" 52 #include "auditor-database/insert_auditor_closure_lags.h" 53 #include "auditor-database/insert_auditor_progress.h" 54 #include "auditor-database/insert_balance.h" 55 #include "auditor-database/insert_row_inconsistency.h" 56 #include "auditor-database/insert_row_minor_inconsistencies.h" 57 #include "auditor-database/insert_wire_format_inconsistency.h" 58 #include "auditor-database/insert_wire_out_inconsistency.h" 59 #include "auditor-database/preflight.h" 60 #include "auditor-database/start.h" 61 #include "exchange-database/get_profit_drain.h" 62 #include "exchange-database/preflight.h" 63 #include "exchange-database/rollback.h" 64 #include "exchange-database/iterate_reserve_closed_above_serial_id.h" 65 #include "exchange-database/iterate_wire_outs_above_serial_id_by_account.h" 66 #include "exchange-database/start_read_only.h" 67 68 69 /** 70 * Maximum number of wire transfers we process per 71 * (database) transaction. 72 */ 73 #define MAX_PER_TRANSACTION 1024 74 75 /** 76 * How much do we allow the bank and the exchange to disagree about 77 * timestamps? Should be sufficiently large to avoid bogus reports from deltas 78 * created by imperfect clock synchronization and network delay. 79 */ 80 #define TIME_TOLERANCE GNUNET_TIME_relative_multiply ( \ 81 GNUNET_TIME_UNIT_MINUTES, \ 82 15) 83 84 85 /** 86 * How long do we try to long-poll for bank wire transfers? 87 */ 88 #define MAX_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 89 GNUNET_TIME_UNIT_HOURS, \ 90 1) 91 92 93 /** 94 * How long do we wait between polling for bank wire transfers at the minimum? 95 */ 96 #define MIN_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 97 GNUNET_TIME_UNIT_MINUTES, \ 98 5) 99 100 101 /** 102 * Run in test mode. Exit when idle instead of 103 * going to sleep and waiting for more work. 104 */ 105 static int test_mode; 106 107 108 /** 109 * Information we keep for each supported account. 110 */ 111 struct WireAccount 112 { 113 /** 114 * Accounts are kept in a DLL. 115 */ 116 struct WireAccount *next; 117 118 /** 119 * Plugins are kept in a DLL. 120 */ 121 struct WireAccount *prev; 122 123 /** 124 * Account details. 125 */ 126 const struct TALER_EXCHANGEDB_AccountInfo *ai; 127 128 /** 129 * Active wire request for the transaction history. 130 */ 131 struct TALER_BANK_DebitHistoryHandle *dhh; 132 133 /** 134 * Task to trigger @e dhh long-polling. 135 */ 136 struct GNUNET_SCHEDULER_Task *dhh_task; 137 138 /** 139 * Time when we expect the current @e dhh long-poll 140 * to finish and we thus could begin another one. 141 */ 142 struct GNUNET_TIME_Absolute dhh_next; 143 144 /** 145 * Progress point for this account. 146 */ 147 uint64_t last_wire_out_serial_id; 148 149 /** 150 * Initial progress point for this account. 151 */ 152 uint64_t start_wire_out_serial_id; 153 154 /** 155 * Where we are in the outbound transaction history. 156 */ 157 uint64_t wire_off_out; 158 159 /** 160 * Label under which we store our pp's reserve_in_serial_id. 161 */ 162 char *label_wire_out_serial_id; 163 164 /** 165 * Label under which we store our wire_off_out. 166 */ 167 char *label_wire_off_out; 168 }; 169 170 171 /** 172 * Information we track for a reserve being closed. 173 */ 174 struct ReserveClosure 175 { 176 /** 177 * Row in the reserves_closed table for this action. 178 */ 179 uint64_t rowid; 180 181 /** 182 * When was the reserve closed? 183 */ 184 struct GNUNET_TIME_Timestamp execution_date; 185 186 /** 187 * Amount transferred (amount remaining minus fee). 188 */ 189 struct TALER_Amount amount; 190 191 /** 192 * Target account where the money was sent. 193 */ 194 struct TALER_FullPayto receiver_account; 195 196 /** 197 * Wire transfer subject used. 198 */ 199 struct TALER_WireTransferIdentifierRawP wtid; 200 }; 201 202 203 /** 204 * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. 205 */ 206 static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; 207 208 /** 209 * How often do we retry the transaction after a serialization 210 * failure before we give up? 211 */ 212 #define MAX_RETRIES 3 213 214 /** 215 * Return value from main(). 216 */ 217 static int global_ret; 218 219 /** 220 * Number of serialization failures we have retried since the last 221 * successfully committed transaction. 222 */ 223 static unsigned int retries; 224 225 /** 226 * State of the current database transaction with 227 * the auditor DB. 228 */ 229 static enum GNUNET_DB_QueryStatus global_qs; 230 231 /** 232 * Map with information about outgoing wire transfers. 233 * Maps hashes of the wire subjects (in binary encoding) 234 * to `struct ReserveOutInfo`s. 235 */ 236 static struct GNUNET_CONTAINER_MultiHashMap *out_map; 237 238 /** 239 * Head of list of wire accounts we still need to look at. 240 */ 241 static struct WireAccount *wa_head; 242 243 /** 244 * Tail of list of wire accounts we still need to look at. 245 */ 246 static struct WireAccount *wa_tail; 247 248 /** 249 * Last reserve_out / wire_out serial IDs seen. 250 */ 251 static TALER_ARL_DEF_PP (wire_reserve_close_id); 252 253 /** 254 * Total amount that was transferred too much from the exchange. 255 */ 256 static TALER_ARL_DEF_AB (total_bad_amount_out_plus); 257 258 /** 259 * Total amount that was transferred too little from the exchange. 260 */ 261 static TALER_ARL_DEF_AB (total_bad_amount_out_minus); 262 263 /** 264 * Total amount of reserve closures which the exchange did not transfer in time. 265 */ 266 static TALER_ARL_DEF_AB (total_closure_amount_lag); 267 268 /** 269 * Total amount affected by duplicate wire transfer 270 * subjects. 271 */ 272 static TALER_ARL_DEF_AB (wire_debit_duplicate_transfer_subject_total); 273 274 /** 275 * Total amount debited to exchange accounts. 276 */ 277 static TALER_ARL_DEF_AB (total_wire_out); 278 279 /** 280 * Total fees the bank charged the exchange for outgoing wire 281 * transfers, on top of the amounts transferred. 282 */ 283 static TALER_ARL_DEF_AB (total_wire_debit_fees); 284 285 /** 286 * Total amount of profits drained. 287 */ 288 static TALER_ARL_DEF_AB (total_drained); 289 290 /** 291 * Amount of zero in our currency. 292 */ 293 static struct TALER_Amount zero; 294 295 /** 296 * Handle to the context for interacting with the bank. 297 */ 298 static struct GNUNET_CURL_Context *ctx; 299 300 /** 301 * Scheduler context for running the @e ctx. 302 */ 303 static struct GNUNET_CURL_RescheduleContext *rctx; 304 305 /** 306 * Should we run checks that only work for exchange-internal audits? 307 */ 308 static int internal_checks; 309 310 /** 311 * Should we ignore if the bank does not know our bank 312 * account? 313 */ 314 static int ignore_account_404; 315 316 /** 317 * Database event handler to wake us up again. 318 */ 319 static struct GNUNET_DB_EventHandler *eh; 320 321 /** 322 * The auditors's configuration. 323 */ 324 static const struct GNUNET_CONFIGURATION_Handle *cfg; 325 326 327 /** 328 * Entry in map with wire information we expect to obtain from the 329 * #TALER_ARL_edb later. 330 */ 331 struct WireTransferOutInfo 332 { 333 334 /** 335 * Hash of the wire transfer subject. 336 */ 337 struct GNUNET_HashCode subject_hash; 338 339 /** 340 * Expected details about the wire transfer. 341 */ 342 struct TALER_BANK_DebitDetails details; 343 344 }; 345 346 347 /** 348 * Free entry in #out_map. 349 * 350 * @param cls NULL 351 * @param key unused key 352 * @param value the `struct WireTransferOutInfo` to free 353 * @return #GNUNET_OK 354 */ 355 static enum GNUNET_GenericReturnValue 356 free_roi (void *cls, 357 const struct GNUNET_HashCode *key, 358 void *value) 359 { 360 struct WireTransferOutInfo *roi = value; 361 362 (void) cls; 363 GNUNET_assert (GNUNET_YES == 364 GNUNET_CONTAINER_multihashmap_remove (out_map, 365 key, 366 roi)); 367 GNUNET_free (roi); 368 return GNUNET_OK; 369 } 370 371 372 /** 373 * Free entry in #reserve_closures. 374 * 375 * @param cls NULL 376 * @param key unused key 377 * @param value the `struct ReserveClosure` to free 378 * @return #GNUNET_OK 379 */ 380 static enum GNUNET_GenericReturnValue 381 free_rc (void *cls, 382 const struct GNUNET_HashCode *key, 383 void *value) 384 { 385 struct ReserveClosure *rc = value; 386 387 (void) cls; 388 GNUNET_assert (GNUNET_YES == 389 GNUNET_CONTAINER_multihashmap_remove (reserve_closures, 390 key, 391 rc)); 392 GNUNET_free (rc->receiver_account.full_payto); 393 GNUNET_free (rc); 394 return GNUNET_OK; 395 } 396 397 398 /** 399 * Task run on shutdown. 400 * 401 * @param cls NULL 402 */ 403 static void 404 do_shutdown (void *cls) 405 { 406 struct WireAccount *wa; 407 408 (void) cls; 409 if (NULL != eh) 410 { 411 TALER_AUDITORDB_event_listen_cancel (eh); 412 eh = NULL; 413 } 414 TALER_ARL_done (); 415 if (NULL != reserve_closures) 416 { 417 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 418 &free_rc, 419 NULL); 420 GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); 421 reserve_closures = NULL; 422 } 423 if (NULL != out_map) 424 { 425 GNUNET_CONTAINER_multihashmap_iterate (out_map, 426 &free_roi, 427 NULL); 428 GNUNET_CONTAINER_multihashmap_destroy (out_map); 429 out_map = NULL; 430 } 431 while (NULL != (wa = wa_head)) 432 { 433 if (NULL != wa->dhh_task) 434 { 435 GNUNET_SCHEDULER_cancel (wa->dhh_task); 436 wa->dhh_task = NULL; 437 } 438 if (NULL != wa->dhh) 439 { 440 TALER_BANK_debit_history_cancel (wa->dhh); 441 wa->dhh = NULL; 442 } 443 GNUNET_CONTAINER_DLL_remove (wa_head, 444 wa_tail, 445 wa); 446 GNUNET_free (wa->label_wire_out_serial_id); 447 GNUNET_free (wa->label_wire_off_out); 448 GNUNET_free (wa); 449 } 450 if (NULL != ctx) 451 { 452 GNUNET_CURL_fini (ctx); 453 ctx = NULL; 454 } 455 if (NULL != rctx) 456 { 457 GNUNET_CURL_gnunet_rc_destroy (rctx); 458 rctx = NULL; 459 } 460 TALER_EXCHANGEDB_unload_accounts (); 461 TALER_ARL_cfg = NULL; 462 } 463 464 465 /** 466 * Detect any entries in #reserve_closures that were not yet 467 * observed on the wire transfer side and update the progress 468 * point accordingly. 469 * 470 * @param cls NULL 471 * @param key unused key 472 * @param value the `struct ReserveClosure` to free 473 * @return #GNUNET_OK 474 */ 475 static enum GNUNET_GenericReturnValue 476 check_pending_rc (void *cls, 477 const struct GNUNET_HashCode *key, 478 void *value) 479 { 480 struct ReserveClosure *rc = value; 481 482 (void) cls; 483 (void) key; 484 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 485 "Missing wire transfer for closed reserve with balance %s\n", 486 TALER_amount2s (&rc->amount)); 487 if (! TALER_amount_is_zero (&rc->amount)) 488 { 489 struct TALER_AUDITORDB_ClosureLags cl = { 490 .problem_row_id = rc->rowid, 491 .account = rc->receiver_account, 492 .amount = rc->amount, 493 .deadline = rc->execution_date.abs_time, 494 .wtid = rc->wtid 495 }; 496 enum GNUNET_DB_QueryStatus qs; 497 498 qs = TALER_AUDITORDB_insert_auditor_closure_lags ( 499 TALER_ARL_adb, 500 &cl); 501 if (qs < 0) 502 { 503 global_qs = qs; 504 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 505 return GNUNET_SYSERR; 506 } 507 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_closure_amount_lag), 508 &TALER_ARL_USE_AB (total_closure_amount_lag), 509 &rc->amount); 510 } 511 return GNUNET_OK; 512 } 513 514 515 /** 516 * Compute the key under which a reserve closure for a given 517 * @a receiver_account and @a wtid would be stored. 518 * 519 * @param receiver_account payto://-URI of the account 520 * @param wtid wire transfer identifier used 521 * @param[out] key set to the key 522 */ 523 static void 524 hash_rc (const struct TALER_FullPayto receiver_account, 525 const struct TALER_WireTransferIdentifierRawP *wtid, 526 struct GNUNET_HashCode *key) 527 { 528 struct TALER_NormalizedPayto npto 529 = TALER_payto_normalize (receiver_account); 530 size_t slen = strlen (npto.normalized_payto); 531 char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; 532 533 GNUNET_memcpy (buf, 534 wtid, 535 sizeof (*wtid)); 536 GNUNET_memcpy (&buf[sizeof (*wtid)], 537 npto.normalized_payto, 538 slen); 539 GNUNET_CRYPTO_hash (buf, 540 sizeof (buf), 541 key); 542 GNUNET_free (npto.normalized_payto); 543 } 544 545 546 /** 547 * Start the database transactions and begin the audit. 548 * 549 * @return transaction status code 550 */ 551 static enum GNUNET_DB_QueryStatus 552 begin_transaction (void); 553 554 555 /** 556 * Commit the transaction, checkpointing our progress in the auditor DB. 557 * 558 * @param qs transaction status so far 559 */ 560 static void 561 commit (enum GNUNET_DB_QueryStatus qs) 562 { 563 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 564 &check_pending_rc, 565 NULL); 566 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 567 "Transaction logic ended with status %d\n", 568 qs); 569 TALER_EXCHANGEDB_rollback (TALER_ARL_edb); 570 /* A callback that failed recorded its status in #global_qs and 571 stopped its iteration; do not checkpoint past what it skipped. */ 572 if ( (0 <= qs) && 573 (0 > global_qs) ) 574 qs = global_qs; 575 if (0 > qs) 576 goto handle_db_error; 577 qs = TALER_AUDITORDB_insert_balance ( 578 TALER_ARL_adb, 579 TALER_ARL_SET_AB (total_drained), 580 TALER_ARL_SET_AB (total_wire_out), 581 TALER_ARL_SET_AB (total_bad_amount_out_plus), 582 TALER_ARL_SET_AB (total_bad_amount_out_minus), 583 TALER_ARL_SET_AB (total_closure_amount_lag), 584 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 585 TALER_ARL_SET_AB (total_wire_out), 586 TALER_ARL_SET_AB (total_wire_debit_fees), 587 NULL); 588 if (0 > qs) 589 goto handle_db_error; 590 for (struct WireAccount *wa = wa_head; 591 NULL != wa; 592 wa = wa->next) 593 { 594 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 595 "Transaction of account %s ends at %llu/%llu\n", 596 wa->ai->section_name, 597 (unsigned long long) wa->last_wire_out_serial_id, 598 (unsigned long long) wa->wire_off_out); 599 qs = TALER_AUDITORDB_insert_auditor_progress ( 600 TALER_ARL_adb, 601 wa->label_wire_out_serial_id, 602 wa->last_wire_out_serial_id, 603 wa->label_wire_off_out, 604 wa->wire_off_out, 605 NULL); 606 if (0 > qs) 607 goto handle_db_error; 608 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 609 "Transaction ends at %s=%llu for account `%s'\n", 610 wa->label_wire_out_serial_id, 611 (unsigned long long) wa->last_wire_out_serial_id, 612 wa->ai->section_name); 613 } 614 qs = TALER_AUDITORDB_insert_auditor_progress ( 615 TALER_ARL_adb, 616 TALER_ARL_SET_PP (wire_reserve_close_id), 617 NULL); 618 if (0 > qs) 619 goto handle_db_error; 620 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 621 "Concluded audit step at %llu\n", 622 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 623 qs = TALER_AUDITORDB_commit (TALER_ARL_adb); 624 if (0 > qs) 625 goto handle_db_error; 626 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 627 "Transaction concluded!\n"); 628 retries = 0; 629 if (1 == test_mode) 630 GNUNET_SCHEDULER_shutdown (); 631 return; 632 handle_db_error: 633 TALER_AUDITORDB_rollback (TALER_ARL_adb); 634 if (GNUNET_DB_STATUS_SOFT_ERROR == qs) 635 { 636 if (MAX_RETRIES < ++retries) 637 { 638 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 639 "Failed to commit the transaction after %u retries, terminating\n", 640 (unsigned int) MAX_RETRIES); 641 global_ret = EXIT_FAILURE; 642 GNUNET_SCHEDULER_shutdown (); 643 return; 644 } 645 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 646 "Serialization issue, trying again\n"); 647 /* Any bank request still in flight belongs to the attempt we are 648 abandoning. #process_debits() would leave it alone and start 649 nothing, so the restarted audit would never make progress. */ 650 for (struct WireAccount *wa = wa_head; 651 NULL != wa; 652 wa = wa->next) 653 { 654 if (NULL != wa->dhh) 655 { 656 TALER_BANK_debit_history_cancel (wa->dhh); 657 wa->dhh = NULL; 658 } 659 if (NULL != wa->dhh_task) 660 { 661 GNUNET_SCHEDULER_cancel (wa->dhh_task); 662 wa->dhh_task = NULL; 663 } 664 } 665 if (GNUNET_DB_STATUS_HARD_ERROR != begin_transaction ()) 666 return; /* the retry continues asynchronously from here */ 667 } 668 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 669 "Hard database error, terminating\n"); 670 global_ret = EXIT_FAILURE; 671 GNUNET_SCHEDULER_shutdown (); 672 } 673 674 675 /** 676 * Check that @a want is within #TIME_TOLERANCE of @a have. 677 * Otherwise report an inconsistency in row @a rowid of @a table. 678 * 679 * @param table where is the inconsistency (if any) 680 * @param rowid what is the row 681 * @param want what is the expected time 682 * @param have what is the time we got 683 * @return true on success, false to abort 684 */ 685 static bool 686 check_time_difference (const char *table, 687 uint64_t rowid, 688 struct GNUNET_TIME_Timestamp want, 689 struct GNUNET_TIME_Timestamp have) 690 { 691 struct GNUNET_TIME_Relative delta; 692 char *details; 693 694 if (GNUNET_TIME_timestamp_cmp (have, >, want)) 695 delta = GNUNET_TIME_absolute_get_difference (want.abs_time, 696 have.abs_time); 697 else 698 delta = GNUNET_TIME_absolute_get_difference (have.abs_time, 699 want.abs_time); 700 if (GNUNET_TIME_relative_cmp (delta, 701 <=, 702 TIME_TOLERANCE)) 703 return true; 704 705 GNUNET_asprintf (&details, 706 "execution date mismatch (%s)", 707 GNUNET_TIME_relative2s (delta, 708 true)); 709 { 710 struct TALER_AUDITORDB_RowMinorInconsistencies rmi = { 711 .row_table = (char *) table, 712 .problem_row = rowid, 713 .diagnostic = details 714 }; 715 enum GNUNET_DB_QueryStatus qs; 716 717 qs = TALER_AUDITORDB_insert_row_minor_inconsistencies ( 718 TALER_ARL_adb, 719 &rmi); 720 721 if (qs < 0) 722 { 723 global_qs = qs; 724 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 725 GNUNET_free (details); 726 return false; 727 } 728 } 729 GNUNET_free (details); 730 return true; 731 } 732 733 734 /** 735 * Closure for #check_rc_matches 736 */ 737 struct CheckMatchContext 738 { 739 740 /** 741 * Reserve operation looking for a match 742 */ 743 const struct WireTransferOutInfo *roi; 744 745 /** 746 * Set to true if we found a match. 747 */ 748 bool found; 749 }; 750 751 752 /** 753 * Check if any of the reserve closures match the given wire transfer. 754 * 755 * @param[in,out] cls a `struct CheckMatchContext` 756 * @param key key of @a value in #reserve_closures 757 * @param value a `struct ReserveClosure` 758 */ 759 static enum GNUNET_GenericReturnValue 760 check_rc_matches (void *cls, 761 const struct GNUNET_HashCode *key, 762 void *value) 763 { 764 struct CheckMatchContext *cmx = cls; 765 struct ReserveClosure *rc = value; 766 767 if ((0 == GNUNET_memcmp (&cmx->roi->details.wtid, 768 &rc->wtid)) && 769 (0 == TALER_full_payto_cmp (rc->receiver_account, 770 cmx->roi->details.credit_account_uri)) && 771 (0 == TALER_amount_cmp (&rc->amount, 772 &cmx->roi->details.amount))) 773 { 774 if (! check_time_difference ("reserves_closures", 775 rc->rowid, 776 rc->execution_date, 777 cmx->roi->details.execution_date)) 778 { 779 free_rc (NULL, 780 key, 781 rc); 782 return GNUNET_SYSERR; 783 } 784 cmx->found = true; 785 free_rc (NULL, 786 key, 787 rc); 788 return GNUNET_NO; 789 } 790 return GNUNET_OK; 791 } 792 793 794 /** 795 * Maximum required length for make_missing_diag(). 796 */ 797 #define MAX_DIAG_LEN 128 798 799 /** 800 * Make diagnostic string for missing wire transfer. 801 * 802 * @param[out] diag where to write the diagnostic string 803 * @param wtid wire transfer ID to include 804 */ 805 static void 806 make_missing_diag (char diag[MAX_DIAG_LEN], 807 const struct TALER_WireTransferIdentifierRawP *wtid) 808 { 809 char *wtid_s; 810 811 wtid_s = GNUNET_STRINGS_data_to_string_alloc (wtid, 812 sizeof (*wtid)); 813 GNUNET_snprintf (diag, 814 MAX_DIAG_LEN, 815 "expected outgoing wire transfer %s missing", 816 wtid_s); 817 GNUNET_free (wtid_s); 818 } 819 820 821 /** 822 * Check if an existing report on a missing wire 823 * out operation justified the @a roi. If so, 824 * clear the existing report. 825 * 826 * @param roi reserve out operation to check 827 * @return #GNUNET_YES if @a roi was justified by a previous report, 828 * #GNUNET_NO of @a roi was not justified by a previous missing report 829 * #GNUNET_SYSERR on database trouble 830 */ 831 static enum GNUNET_GenericReturnValue 832 check_reported_inconsistency (struct WireTransferOutInfo *roi) 833 { 834 char diag[MAX_DIAG_LEN]; 835 struct TALER_AUDITORDB_WireOutInconsistency woi = { 836 .wire_out_row_id = roi->details.serial_id, 837 .destination_account = roi->details.credit_account_uri, 838 .diagnostic = diag, 839 .expected = roi->details.amount, 840 .claimed = zero, 841 }; 842 enum GNUNET_DB_QueryStatus qs; 843 844 make_missing_diag (diag, 845 &roi->details.wtid); 846 qs = TALER_AUDITORDB_delete_wire_out_inconsistency_if_matching ( 847 TALER_ARL_adb, 848 &woi); 849 if (qs < 0) 850 { 851 global_qs = qs; 852 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 853 return GNUNET_SYSERR; 854 } 855 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 856 { 857 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 858 "Deletion of wire out inconsistency %llu (%s, %s, %s) failed: not reported missing!\n", 859 (unsigned long long) roi->details.serial_id, 860 roi->details.credit_account_uri.full_payto, 861 diag, 862 TALER_amount2s (&roi->details.amount)); 863 return GNUNET_NO; 864 } 865 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 866 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 867 &roi->details.amount); 868 return GNUNET_YES; 869 } 870 871 872 /** 873 * Check if a profit drain operation justified the @a roi 874 * 875 * @param roi reserve out operation to check 876 * @return #GNUNET_YES if @a roi was justified by a profit drain, 877 * #GNUNET_NO of @a roi was not justified by a proft drain 878 * #GNUNET_SYSERR on database trouble 879 */ 880 static enum GNUNET_GenericReturnValue 881 check_profit_drain (struct WireTransferOutInfo *roi) 882 { 883 enum GNUNET_DB_QueryStatus qs; 884 uint64_t serial; 885 char *account_section; 886 struct TALER_FullPayto payto_uri; 887 struct GNUNET_TIME_Timestamp request_timestamp; 888 struct TALER_Amount amount; 889 struct TALER_MasterSignatureP master_sig; 890 891 qs = TALER_EXCHANGEDB_get_profit_drain ( 892 TALER_ARL_edb, 893 &roi->details.wtid, 894 &serial, 895 &account_section, 896 &payto_uri, 897 &request_timestamp, 898 &amount, 899 &master_sig); 900 switch (qs) 901 { 902 case GNUNET_DB_STATUS_HARD_ERROR: 903 GNUNET_break (0); 904 global_ret = EXIT_FAILURE; 905 GNUNET_SCHEDULER_shutdown (); 906 return GNUNET_SYSERR; 907 case GNUNET_DB_STATUS_SOFT_ERROR: 908 /* should fail on commit later ... */ 909 GNUNET_break (0); 910 return GNUNET_SYSERR; 911 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 912 /* not a profit drain */ 913 return GNUNET_NO; 914 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 915 break; 916 } 917 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 918 "Profit drain of %s to %s found!\n", 919 TALER_amount2s (&amount), 920 payto_uri.full_payto); 921 if (GNUNET_OK != 922 TALER_exchange_offline_profit_drain_verify ( 923 &roi->details.wtid, 924 request_timestamp, 925 &amount, 926 account_section, 927 payto_uri, 928 &TALER_ARL_master_pub, 929 &master_sig)) 930 { 931 struct TALER_AUDITORDB_RowInconsistency ri = { 932 .problem_row_id = roi->details.serial_id, 933 .row_table = (char *) "profit_drains", 934 .diagnostic = (char *) "invalid signature" 935 }; 936 937 GNUNET_break (0); 938 qs = TALER_AUDITORDB_insert_row_inconsistency ( 939 TALER_ARL_adb, 940 &ri); 941 GNUNET_free (payto_uri.full_payto); 942 GNUNET_free (account_section); 943 if (qs < 0) 944 { 945 global_qs = qs; 946 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 947 return GNUNET_SYSERR; 948 } 949 return GNUNET_NO; 950 } 951 GNUNET_free (account_section); 952 953 { 954 if (0 != 955 TALER_full_payto_normalize_and_cmp (payto_uri, 956 roi->details.credit_account_uri)) 957 { 958 struct TALER_AUDITORDB_WireOutInconsistency woi = { 959 .wire_out_row_id = serial, 960 .destination_account = roi->details.credit_account_uri, 961 .diagnostic = (char *) "profit drain wired to invalid account", 962 .expected = roi->details.amount, 963 .claimed = zero, 964 }; 965 966 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 967 TALER_ARL_adb, 968 &woi); 969 if (qs < 0) 970 { 971 global_qs = qs; 972 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 973 GNUNET_free (payto_uri.full_payto); 974 return GNUNET_SYSERR; 975 } 976 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 977 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 978 &amount); 979 GNUNET_free (payto_uri.full_payto); 980 return GNUNET_YES; /* justified, kind-of */ 981 } 982 } 983 GNUNET_free (payto_uri.full_payto); 984 if (0 != 985 TALER_amount_cmp (&amount, 986 &roi->details.amount)) 987 { 988 struct TALER_AUDITORDB_WireOutInconsistency woi = { 989 .wire_out_row_id = roi->details.serial_id, 990 .destination_account = roi->details.credit_account_uri, 991 .diagnostic = (char *) "incorrect amount drained to correct account", 992 .expected = roi->details.amount, 993 .claimed = amount, 994 }; 995 996 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 997 TALER_ARL_adb, 998 &woi); 999 if (qs < 0) 1000 { 1001 global_qs = qs; 1002 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1003 return GNUNET_SYSERR; 1004 } 1005 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1006 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1007 &roi->details.amount); 1008 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1009 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1010 &amount); 1011 return GNUNET_YES; /* justified, kind-of */ 1012 } 1013 /* profit drain was correct */ 1014 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_drained), 1015 &TALER_ARL_USE_AB (total_drained), 1016 &amount); 1017 return GNUNET_YES; 1018 } 1019 1020 1021 /** 1022 * Check whether the given transfer was justified because it was 1023 * actually previously reported as missing. 1024 * 1025 * @param roi the reserve out operation to check 1026 * @return #GNUNET_OK on success, #GNUNET_NO if there was no 1027 * matching lag, #GNUNET_SYSERR on database trouble 1028 */ 1029 static enum GNUNET_GenericReturnValue 1030 check_closure_lag (const struct WireTransferOutInfo *roi) 1031 { 1032 enum GNUNET_DB_QueryStatus qs; 1033 1034 qs = TALER_AUDITORDB_delete_auditor_closure_lag ( 1035 TALER_ARL_adb, 1036 &roi->details.amount, 1037 &roi->details.wtid, 1038 roi->details.credit_account_uri); 1039 if (qs < 0) 1040 { 1041 global_qs = qs; 1042 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1043 return GNUNET_SYSERR; 1044 } 1045 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1046 return GNUNET_NO; 1047 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1048 "Cleared closure lag: found justification\n"); 1049 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_closure_amount_lag), 1050 &TALER_ARL_USE_AB (total_closure_amount_lag), 1051 &roi->details.amount); 1052 return GNUNET_YES; /* found! */ 1053 } 1054 1055 1056 /** 1057 * Check whether the given transfer was justified by a reserve closure or 1058 * profit drain. If not, complain that we failed to match an entry from 1059 * #out_map. This means a wire transfer was made without proper 1060 * justification. 1061 * 1062 * @param cls a `struct WireAccount` 1063 * @param key unused key 1064 * @param value the `struct WireTransferOutInfo` to report 1065 * @return #GNUNET_OK on success 1066 */ 1067 static enum GNUNET_GenericReturnValue 1068 complain_out_not_found (void *cls, 1069 const struct GNUNET_HashCode *key, 1070 void *value) 1071 { 1072 // struct WireAccount *wa = cls; 1073 struct WireTransferOutInfo *roi = value; 1074 struct GNUNET_HashCode rkey; 1075 struct CheckMatchContext cmx = { 1076 .roi = roi, 1077 .found = false 1078 }; 1079 enum GNUNET_GenericReturnValue ret; 1080 1081 (void) cls; 1082 (void) key; 1083 hash_rc (roi->details.credit_account_uri, 1084 &roi->details.wtid, 1085 &rkey); 1086 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1087 "Checking for reserve closure %s benefiting %s\n", 1088 GNUNET_h2s (&rkey), 1089 roi->details.credit_account_uri.full_payto); 1090 GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, 1091 &rkey, 1092 &check_rc_matches, 1093 &cmx); 1094 if (cmx.found) 1095 return GNUNET_OK; 1096 ret = check_reported_inconsistency (roi); 1097 if (GNUNET_NO != ret) 1098 return ret; 1099 ret = check_profit_drain (roi); 1100 if (GNUNET_NO != ret) 1101 return ret; 1102 ret = check_closure_lag (roi); 1103 if (GNUNET_NO != ret) 1104 return ret; 1105 1106 { 1107 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1108 .destination_account = roi->details.credit_account_uri, 1109 .diagnostic = (char *) "missing justification for outgoing wire transfer", 1110 .wire_out_row_id = roi->details.serial_id, 1111 .expected = zero, 1112 .claimed = roi->details.amount 1113 }; 1114 enum GNUNET_DB_QueryStatus qs; 1115 1116 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1117 TALER_ARL_adb, 1118 &woi); 1119 if (qs < 0) 1120 { 1121 global_qs = qs; 1122 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1123 return GNUNET_SYSERR; 1124 } 1125 } 1126 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1127 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1128 &roi->details.amount); 1129 return GNUNET_OK; 1130 } 1131 1132 1133 /** 1134 * Function called with details about outgoing wire transfers 1135 * as claimed by the exchange DB. 1136 * 1137 * @param wa a `struct WireAccount` 1138 * @param rowid unique serial ID in wire_out table 1139 * @param date timestamp of the transfer (roughly) 1140 * @param wtid wire transfer subject 1141 * @param payto_uri wire transfer details of the receiver 1142 * @param amount amount that was wired 1143 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1144 */ 1145 static enum GNUNET_GenericReturnValue 1146 wire_out_cb ( 1147 struct WireAccount *wa, 1148 uint64_t rowid, 1149 struct GNUNET_TIME_Timestamp date, 1150 const struct TALER_WireTransferIdentifierRawP *wtid, 1151 const struct TALER_FullPayto payto_uri, 1152 const struct TALER_Amount *amount) 1153 { 1154 struct GNUNET_HashCode key; 1155 struct WireTransferOutInfo *roi; 1156 1157 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1158 "Exchange wire OUT #%llu at %s of %s with WTID %s\n", 1159 (unsigned long long) rowid, 1160 GNUNET_TIME_timestamp2s (date), 1161 TALER_amount2s (amount), 1162 TALER_B2S (wtid)); 1163 wa->last_wire_out_serial_id = rowid + 1; 1164 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_wire_out), 1165 &TALER_ARL_USE_AB (total_wire_out), 1166 amount); 1167 GNUNET_CRYPTO_hash (wtid, 1168 sizeof (*wtid), 1169 &key); 1170 roi = GNUNET_CONTAINER_multihashmap_get (out_map, 1171 &key); 1172 if (NULL == roi) 1173 { 1174 /* Wire transfer was not made (yet) at all (but would have been 1175 justified), so the entire amount is missing / still to be done. This 1176 is moderately harmless, it might just be that the 1177 taler-exchange-transfer tool or bank has not yet fully caught up with 1178 the transfers it should do. 1179 May be cleared later by check_reported_inconsistency() */ 1180 char diag[MAX_DIAG_LEN]; 1181 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1182 .destination_account = payto_uri, 1183 .diagnostic = diag, 1184 .wire_out_row_id = rowid, 1185 .expected = *amount, 1186 .claimed = zero, 1187 }; 1188 enum GNUNET_DB_QueryStatus qs; 1189 1190 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1191 "Wire out for row %llu still missing\n", 1192 (unsigned long long) rowid); 1193 make_missing_diag (diag, 1194 wtid); 1195 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1196 TALER_ARL_adb, 1197 &woi); 1198 if (qs < 0) 1199 { 1200 global_qs = qs; 1201 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1202 return GNUNET_SYSERR; 1203 } 1204 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1205 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1206 amount); 1207 return GNUNET_OK; 1208 } 1209 1210 if (0 != TALER_full_payto_normalize_and_cmp (payto_uri, 1211 roi->details.credit_account_uri)) 1212 { 1213 /* Destination bank account is wrong in actual wire transfer, so 1214 we should count the wire transfer as entirely spurious, and 1215 additionally consider the justified wire transfer as missing. */ 1216 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1217 .wire_out_row_id = rowid, 1218 .destination_account = payto_uri, 1219 .diagnostic = (char *) "receiver account mismatch", 1220 .expected = *amount, 1221 .claimed = roi->details.amount, 1222 }; 1223 enum GNUNET_DB_QueryStatus qs; 1224 1225 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1226 TALER_ARL_adb, 1227 &woi); 1228 if (qs < 0) 1229 { 1230 global_qs = qs; 1231 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1232 return GNUNET_SYSERR; 1233 } 1234 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1235 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1236 &roi->details.amount); 1237 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1238 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1239 amount); 1240 GNUNET_assert (GNUNET_OK == 1241 free_roi (NULL, 1242 &key, 1243 roi)); 1244 return GNUNET_OK; 1245 } 1246 1247 if (0 != TALER_amount_cmp (&roi->details.amount, 1248 amount)) 1249 { 1250 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1251 .destination_account = payto_uri, 1252 .diagnostic = (char *) "wire amount does not match", 1253 .wire_out_row_id = rowid, 1254 .expected = *amount, 1255 .claimed = roi->details.amount, 1256 }; 1257 enum GNUNET_DB_QueryStatus qs; 1258 1259 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1260 TALER_ARL_adb, 1261 &woi); 1262 if (qs < 0) 1263 { 1264 global_qs = qs; 1265 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1266 return GNUNET_SYSERR; 1267 } 1268 if (0 < TALER_amount_cmp (amount, 1269 &roi->details.amount)) 1270 { 1271 /* amount > roi->details.amount: wire transfer was smaller than it should have been */ 1272 struct TALER_Amount delta; 1273 1274 TALER_ARL_amount_subtract (&delta, 1275 amount, 1276 &roi->details.amount); 1277 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1278 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1279 &delta); 1280 } 1281 else 1282 { 1283 /* roi->details.amount < amount: wire transfer was larger than it should have been */ 1284 struct TALER_Amount delta; 1285 1286 TALER_ARL_amount_subtract (&delta, 1287 &roi->details.amount, 1288 amount); 1289 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1290 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1291 &delta); 1292 } 1293 GNUNET_assert (GNUNET_OK == 1294 free_roi (NULL, 1295 &key, 1296 roi)); 1297 return GNUNET_OK; 1298 } 1299 1300 { 1301 enum GNUNET_GenericReturnValue ret; 1302 1303 if (! check_time_difference ("wire_out", 1304 rowid, 1305 date, 1306 roi->details.execution_date)) 1307 { 1308 /* We had a database error, fail */ 1309 ret = GNUNET_SYSERR; 1310 } 1311 else 1312 { 1313 ret = GNUNET_OK; 1314 } 1315 GNUNET_assert (GNUNET_OK == 1316 free_roi (NULL, 1317 &key, 1318 roi)); 1319 return ret; 1320 } 1321 } 1322 1323 1324 /** 1325 * Main function for processing 'debit' data. We start by going over 1326 * the DEBIT transactions this time, and then verify that all of them are 1327 * justified by reserve closures, profit drains or regular outgoing 1328 * wire transfers from aggregated deposits. 1329 * 1330 * @param[in,out] wa wire account list to process 1331 */ 1332 static void 1333 process_debits (struct WireAccount *wa); 1334 1335 1336 /** 1337 * Go over the "wire_out" table of the exchange and 1338 * verify that all wire outs are in that table. 1339 * 1340 * @param[in,out] wa wire account we are processing 1341 */ 1342 static void 1343 check_exchange_wire_out (struct WireAccount *wa) 1344 { 1345 enum GNUNET_DB_QueryStatus qs; 1346 1347 GNUNET_assert (NULL == wa->dhh); 1348 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1349 "Analyzing exchange's wire OUT table for account `%s'\n", 1350 wa->ai->section_name); 1351 qs = TALER_EXCHANGEDB_iterate_wire_outs_above_serial_id_by_account ( 1352 TALER_ARL_edb, 1353 wa->ai->section_name, 1354 wa->last_wire_out_serial_id, 1355 &wire_out_cb, 1356 wa); 1357 if (0 > qs) 1358 { 1359 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1360 global_ret = EXIT_FAILURE; 1361 GNUNET_SCHEDULER_shutdown (); 1362 return; 1363 } 1364 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1365 &complain_out_not_found, 1366 wa); 1367 /* clean up */ 1368 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1369 &free_roi, 1370 NULL); 1371 process_debits (wa->next); 1372 } 1373 1374 1375 /** 1376 * This function is called for all transactions that 1377 * are debited from the exchange's account (outgoing 1378 * transactions). 1379 * 1380 * @param cls `struct WireAccount` with current wire account to process 1381 * @param dhr HTTP response details 1382 */ 1383 static void 1384 history_debit_cb ( 1385 void *cls, 1386 const struct TALER_BANK_DebitHistoryResponse *dhr); 1387 1388 1389 /** 1390 * Task scheduled to begin long-polling on the 1391 * bank transfer. 1392 * 1393 * @param cls a `struct WireAccount *` 1394 */ 1395 static void 1396 dh_long_poll (void *cls) 1397 { 1398 struct WireAccount *wa = cls; 1399 1400 wa->dhh_task = NULL; 1401 wa->dhh_next 1402 = GNUNET_TIME_relative_to_absolute (MIN_LONGPOLL_DELAY); 1403 GNUNET_assert (NULL == wa->dhh); 1404 wa->dhh = TALER_BANK_debit_history ( 1405 ctx, 1406 wa->ai->auth, 1407 wa->wire_off_out, 1408 MAX_PER_TRANSACTION, 1409 MAX_LONGPOLL_DELAY, 1410 &history_debit_cb, 1411 wa); 1412 if (NULL == wa->dhh) 1413 { 1414 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1415 "Failed to start long-polling for bank transaction history for `%s'\n", 1416 wa->ai->section_name); 1417 global_ret = EXIT_FAILURE; 1418 GNUNET_SCHEDULER_shutdown (); 1419 return; 1420 } 1421 } 1422 1423 1424 static void 1425 history_debit_cb ( 1426 void *cls, 1427 const struct TALER_BANK_DebitHistoryResponse *dhr) 1428 { 1429 struct WireAccount *wa = cls; 1430 struct WireTransferOutInfo *roi; 1431 size_t slen; 1432 1433 wa->dhh = NULL; 1434 if ( (MHD_HTTP_OK == dhr->http_status) && 1435 (0 != dhr->details.ok.details_length) ) 1436 { 1437 /* As we got results, we go again *immediately* */ 1438 wa->dhh_next = GNUNET_TIME_UNIT_ZERO_ABS; 1439 } 1440 GNUNET_assert (NULL == wa->dhh_task); 1441 wa->dhh_task 1442 = GNUNET_SCHEDULER_add_at (wa->dhh_next, 1443 &dh_long_poll, 1444 wa); 1445 switch (dhr->http_status) 1446 { 1447 case MHD_HTTP_OK: 1448 for (unsigned int i = 0; i < dhr->details.ok.details_length; i++) 1449 { 1450 const struct TALER_BANK_DebitDetails *dd 1451 = &dhr->details.ok.details[i]; 1452 1453 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1454 "Analyzing bank DEBIT #%llu at %s of %s with WTID %s\n", 1455 (unsigned long long) dd->serial_id, 1456 GNUNET_TIME_timestamp2s (dd->execution_date), 1457 TALER_amount2s (&dd->amount), 1458 TALER_B2S (&dd->wtid)); 1459 wa->wire_off_out = dd->serial_id; 1460 /* dd->amount is what was transferred, and what the exchange's 1461 justification (wire_out, reserve closure, profit drain) must 1462 match; the bank's fee was debited on top of it. */ 1463 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_wire_debit_fees), 1464 &TALER_ARL_USE_AB (total_wire_debit_fees), 1465 &dd->debit_fee); 1466 slen = strlen (dd->credit_account_uri.full_payto) + 1; 1467 roi = GNUNET_malloc (sizeof (struct WireTransferOutInfo) 1468 + slen); 1469 GNUNET_CRYPTO_hash (&dd->wtid, 1470 sizeof (dd->wtid), 1471 &roi->subject_hash); 1472 roi->details = *dd; 1473 roi->details.credit_account_uri.full_payto 1474 = (char *) &roi[1]; 1475 GNUNET_memcpy (&roi[1], 1476 dd->credit_account_uri.full_payto, 1477 slen); 1478 if (GNUNET_OK != 1479 GNUNET_CONTAINER_multihashmap_put (out_map, 1480 &roi->subject_hash, 1481 roi, 1482 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) 1483 { 1484 struct TALER_AUDITORDB_WireFormatInconsistency wfi = { 1485 .amount = dd->amount, 1486 .wire_offset = dd->serial_id, 1487 .diagnostic = (char *) "duplicate outgoing wire transfer subject" 1488 }; 1489 enum GNUNET_DB_QueryStatus qs; 1490 1491 qs = TALER_AUDITORDB_insert_wire_format_inconsistency ( 1492 TALER_ARL_adb, 1493 &wfi); 1494 if (qs < 0) 1495 { 1496 global_qs = qs; 1497 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1498 commit (qs); 1499 return; 1500 } 1501 TALER_ARL_amount_add (&TALER_ARL_USE_AB ( 1502 wire_debit_duplicate_transfer_subject_total), 1503 &TALER_ARL_USE_AB ( 1504 wire_debit_duplicate_transfer_subject_total), 1505 &dd->amount); 1506 } 1507 } 1508 check_exchange_wire_out (wa); 1509 return; 1510 case MHD_HTTP_NO_CONTENT: 1511 check_exchange_wire_out (wa); 1512 return; 1513 case MHD_HTTP_NOT_FOUND: 1514 if (ignore_account_404) 1515 { 1516 check_exchange_wire_out (wa); 1517 return; 1518 } 1519 break; 1520 default: 1521 break; 1522 } 1523 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1524 "Error fetching debit history of account %s: %u/%u!\n", 1525 wa->ai->section_name, 1526 dhr->http_status, 1527 (unsigned int) dhr->ec); 1528 commit (GNUNET_DB_STATUS_HARD_ERROR); 1529 global_ret = EXIT_FAILURE; 1530 GNUNET_SCHEDULER_shutdown (); 1531 return; 1532 } 1533 1534 1535 static void 1536 process_debits (struct WireAccount *wa) 1537 { 1538 /* skip accounts where DEBIT is not enabled */ 1539 while ( (NULL != wa) && 1540 (! wa->ai->debit_enabled) ) 1541 wa = wa->next; 1542 if (NULL == wa) 1543 { 1544 /* end of iteration */ 1545 commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); 1546 return; 1547 } 1548 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1549 "Checking bank DEBIT records of account `%s'\n", 1550 wa->ai->section_name); 1551 if ( (NULL == wa->dhh) && 1552 (NULL == wa->dhh_task) ) 1553 { 1554 wa->dhh = TALER_BANK_debit_history ( 1555 ctx, 1556 wa->ai->auth, 1557 wa->wire_off_out, 1558 MAX_PER_TRANSACTION, 1559 GNUNET_TIME_UNIT_ZERO, 1560 &history_debit_cb, 1561 wa); 1562 if (NULL == wa->dhh) 1563 { 1564 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1565 "Failed to obtain bank transaction history for `%s'\n", 1566 wa->ai->section_name); 1567 commit (GNUNET_DB_STATUS_HARD_ERROR); 1568 global_ret = EXIT_FAILURE; 1569 GNUNET_SCHEDULER_shutdown (); 1570 return; 1571 } 1572 } 1573 } 1574 1575 1576 /** 1577 * Function called about reserve closing operations the aggregator triggered. 1578 * 1579 * @param cls closure; NULL 1580 * @param rowid row identifier used to uniquely identify the reserve closing operation 1581 * @param execution_date when did we execute the close operation 1582 * @param amount_with_fee how much did we debit the reserve 1583 * @param closing_fee how much did we charge for closing the reserve 1584 * @param reserve_pub public key of the reserve 1585 * @param receiver_account where did we send the funds, in payto://-format 1586 * @param wtid identifier used for the wire transfer 1587 * @param close_request_row which close request triggered the operation? 1588 * 0 if it was a timeout (not used) 1589 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1590 */ 1591 static enum GNUNET_GenericReturnValue 1592 reserve_closed_cb ( 1593 void *cls, 1594 uint64_t rowid, 1595 struct GNUNET_TIME_Timestamp execution_date, 1596 const struct TALER_Amount *amount_with_fee, 1597 const struct TALER_Amount *closing_fee, 1598 const struct TALER_ReservePublicKeyP *reserve_pub, 1599 const struct TALER_FullPayto receiver_account, 1600 const struct TALER_WireTransferIdentifierRawP *wtid, 1601 uint64_t close_request_row) 1602 { 1603 struct ReserveClosure *rc; 1604 struct GNUNET_HashCode key; 1605 1606 (void) cls; 1607 (void) close_request_row; 1608 GNUNET_assert (TALER_ARL_USE_PP (wire_reserve_close_id) <= rowid); 1609 TALER_ARL_USE_PP (wire_reserve_close_id) = rowid + 1; 1610 rc = GNUNET_new (struct ReserveClosure); 1611 if (TALER_ARL_SR_INVALID_NEGATIVE == 1612 TALER_ARL_amount_subtract_neg (&rc->amount, 1613 amount_with_fee, 1614 closing_fee)) 1615 { 1616 struct TALER_AUDITORDB_RowInconsistency ri = { 1617 .problem_row_id = rowid, 1618 .row_table 1619 = (char *) "reserves_closures", 1620 .diagnostic 1621 = (char *) "closing fee above reserve balance (and closed anyway)" 1622 }; 1623 enum GNUNET_DB_QueryStatus qs; 1624 1625 qs = TALER_AUDITORDB_insert_row_inconsistency ( 1626 TALER_ARL_adb, 1627 &ri); 1628 if (qs < 0) 1629 { 1630 global_qs = qs; 1631 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1632 return GNUNET_OK; 1633 } 1634 GNUNET_free (rc); 1635 return GNUNET_OK; 1636 } 1637 rc->receiver_account.full_payto 1638 = GNUNET_strdup (receiver_account.full_payto); 1639 rc->wtid = *wtid; 1640 rc->execution_date = execution_date; 1641 rc->rowid = rowid; 1642 hash_rc (rc->receiver_account, 1643 wtid, 1644 &key); 1645 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1646 "Discovered reserve closure %llu (%s) over %s benefiting %s\n", 1647 (unsigned long long) rowid, 1648 GNUNET_h2s (&key), 1649 TALER_amount2s (amount_with_fee), 1650 receiver_account.full_payto); 1651 (void) GNUNET_CONTAINER_multihashmap_put ( 1652 reserve_closures, 1653 &key, 1654 rc, 1655 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); 1656 return GNUNET_OK; 1657 } 1658 1659 1660 /** 1661 * Start the database transactions and begin the audit. 1662 * 1663 * @return transaction status code 1664 */ 1665 static enum GNUNET_DB_QueryStatus 1666 begin_transaction (void) 1667 { 1668 enum GNUNET_DB_QueryStatus qs; 1669 1670 if (GNUNET_SYSERR == 1671 TALER_EXCHANGEDB_preflight (TALER_ARL_edb)) 1672 { 1673 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1674 "Failed to initialize exchange database connection.\n"); 1675 return GNUNET_DB_STATUS_HARD_ERROR; 1676 } 1677 if (GNUNET_SYSERR == 1678 TALER_AUDITORDB_preflight (TALER_ARL_adb)) 1679 { 1680 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1681 "Failed to initialize auditor database session.\n"); 1682 return GNUNET_DB_STATUS_HARD_ERROR; 1683 } 1684 global_qs = GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1685 if (GNUNET_OK != 1686 TALER_AUDITORDB_start (TALER_ARL_adb, 1687 "auditor-wire-debit")) 1688 { 1689 GNUNET_break (0); 1690 return GNUNET_DB_STATUS_HARD_ERROR; 1691 } 1692 if (GNUNET_OK != 1693 TALER_EXCHANGEDB_start_read_only (TALER_ARL_edb, 1694 "wire debit auditor")) 1695 { 1696 GNUNET_break (0); 1697 return GNUNET_DB_STATUS_HARD_ERROR; 1698 } 1699 qs = TALER_AUDITORDB_get_balance ( 1700 TALER_ARL_adb, 1701 TALER_ARL_GET_AB (total_drained), 1702 TALER_ARL_GET_AB (total_wire_out), 1703 TALER_ARL_GET_AB (total_bad_amount_out_plus), 1704 TALER_ARL_GET_AB (total_bad_amount_out_minus), 1705 TALER_ARL_GET_AB (total_closure_amount_lag), 1706 TALER_ARL_GET_AB (wire_debit_duplicate_transfer_subject_total), 1707 TALER_ARL_GET_AB (total_wire_out), 1708 TALER_ARL_GET_AB (total_wire_debit_fees), 1709 NULL); 1710 switch (qs) 1711 { 1712 case GNUNET_DB_STATUS_HARD_ERROR: 1713 GNUNET_break (0); 1714 return qs; 1715 case GNUNET_DB_STATUS_SOFT_ERROR: 1716 GNUNET_break (0); 1717 return qs; 1718 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 1719 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 1720 break; 1721 } 1722 for (struct WireAccount *wa = wa_head; 1723 NULL != wa; 1724 wa = wa->next) 1725 { 1726 /* we are called again after a serialization failure, so the labels 1727 may already have been computed */ 1728 GNUNET_free (wa->label_wire_out_serial_id); 1729 GNUNET_free (wa->label_wire_off_out); 1730 GNUNET_asprintf (&wa->label_wire_out_serial_id, 1731 "wire-%s-%s", 1732 wa->ai->section_name, 1733 "wire_out_serial_id"); 1734 GNUNET_asprintf (&wa->label_wire_off_out, 1735 "wire-%s-%s", 1736 wa->ai->section_name, 1737 "wire_off_out"); 1738 qs = TALER_AUDITORDB_get_auditor_progress ( 1739 TALER_ARL_adb, 1740 wa->label_wire_out_serial_id, 1741 &wa->last_wire_out_serial_id, 1742 wa->label_wire_off_out, 1743 &wa->wire_off_out, 1744 NULL); 1745 if (0 > qs) 1746 { 1747 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1748 return qs; 1749 } 1750 GNUNET_assert (2 == qs); 1751 wa->start_wire_out_serial_id = wa->last_wire_out_serial_id; 1752 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1753 "Resuming account %s debit audit at %llu/%llu\n", 1754 wa->ai->section_name, 1755 (unsigned long long) wa->last_wire_out_serial_id, 1756 (unsigned long long) wa->wire_off_out); 1757 } 1758 qs = TALER_AUDITORDB_get_auditor_progress ( 1759 TALER_ARL_adb, 1760 TALER_ARL_GET_PP (wire_reserve_close_id), 1761 NULL); 1762 if (0 > qs) 1763 { 1764 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1765 return qs; 1766 } 1767 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1768 { 1769 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, 1770 "First analysis of with wire auditor, starting audit from scratch\n"); 1771 } 1772 else 1773 { 1774 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1775 "Resuming wire debit audit at %llu\n", 1776 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1777 } 1778 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1779 "Iterating over reserve closures from %llu\n", 1780 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1781 qs = TALER_EXCHANGEDB_iterate_reserve_closed_above_serial_id ( 1782 TALER_ARL_edb, 1783 TALER_ARL_USE_PP (wire_reserve_close_id), 1784 &reserve_closed_cb, 1785 NULL); 1786 if (0 > qs) 1787 { 1788 GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); 1789 return GNUNET_DB_STATUS_HARD_ERROR; 1790 } 1791 process_debits (wa_head); 1792 return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1793 } 1794 1795 1796 /** 1797 * Function called with information about a wire account. Adds the 1798 * account to our list for processing (if it is enabled and we can 1799 * load the plugin). 1800 * 1801 * @param cls closure, NULL 1802 * @param ai account information 1803 */ 1804 static void 1805 process_account_cb (void *cls, 1806 const struct TALER_EXCHANGEDB_AccountInfo *ai) 1807 { 1808 struct WireAccount *wa; 1809 1810 (void) cls; 1811 if ( (! ai->debit_enabled) && 1812 (! ai->credit_enabled) ) 1813 return; /* not an active exchange account */ 1814 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1815 "Found exchange account `%s'\n", 1816 ai->section_name); 1817 wa = GNUNET_new (struct WireAccount); 1818 wa->ai = ai; 1819 GNUNET_CONTAINER_DLL_insert (wa_head, 1820 wa_tail, 1821 wa); 1822 } 1823 1824 1825 /** 1826 * Function called on events received from Postgres. 1827 * 1828 * @param cls closure, NULL 1829 * @param extra additional event data provided 1830 * @param extra_size number of bytes in @a extra 1831 */ 1832 static void 1833 db_notify (void *cls, 1834 const void *extra, 1835 size_t extra_size) 1836 { 1837 (void) cls; 1838 (void) extra; 1839 (void) extra_size; 1840 1841 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1842 "Received notification to wake wire helper\n"); 1843 /* If there are accounts we are still processing, abort 1844 the HTTP requests so we can start afresh. */ 1845 for (struct WireAccount *wa = wa_head; 1846 NULL != wa; 1847 wa = wa->next) 1848 { 1849 if (NULL != wa->dhh) 1850 { 1851 TALER_BANK_debit_history_cancel (wa->dhh); 1852 wa->dhh = NULL; 1853 } 1854 check_exchange_wire_out (wa); 1855 } 1856 1857 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1858 begin_transaction ()) 1859 { 1860 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1861 "Audit failed\n"); 1862 GNUNET_break (0); 1863 global_ret = EXIT_FAILURE; 1864 GNUNET_SCHEDULER_shutdown (); 1865 return; 1866 } 1867 } 1868 1869 1870 /** 1871 * Main function that will be run. 1872 * 1873 * @param cls closure 1874 * @param args remaining command-line arguments 1875 * @param cfgfile name of the configuration file used (for saving, can be NULL!) 1876 * @param c configuration 1877 */ 1878 static void 1879 run (void *cls, 1880 char *const *args, 1881 const char *cfgfile, 1882 const struct GNUNET_CONFIGURATION_Handle *c) 1883 { 1884 (void) cls; 1885 (void) args; 1886 (void) cfgfile; 1887 cfg = c; 1888 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1889 "Launching wire debit auditor\n"); 1890 if (EXIT_SUCCESS != 1891 (global_ret = TALER_ARL_init (c))) 1892 { 1893 return; 1894 } 1895 1896 reserve_closures 1897 = GNUNET_CONTAINER_multihashmap_create (1024, 1898 GNUNET_NO); 1899 GNUNET_assert (GNUNET_OK == 1900 TALER_amount_set_zero (TALER_ARL_currency, 1901 &zero)); 1902 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 1903 NULL); 1904 ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, 1905 &rctx); 1906 rctx = GNUNET_CURL_gnunet_rc_create (ctx); 1907 if (NULL == ctx) 1908 { 1909 GNUNET_break (0); 1910 global_ret = EXIT_FAILURE; 1911 return; 1912 } 1913 reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, 1914 GNUNET_NO); 1915 out_map = GNUNET_CONTAINER_multihashmap_create (1024, 1916 true); 1917 if (GNUNET_OK != 1918 TALER_EXCHANGEDB_load_accounts (TALER_ARL_cfg, 1919 TALER_EXCHANGEDB_ALO_DEBIT 1920 | TALER_EXCHANGEDB_ALO_CREDIT 1921 | TALER_EXCHANGEDB_ALO_AUTHDATA)) 1922 { 1923 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1924 "No bank accounts configured\n"); 1925 global_ret = EXIT_NOTCONFIGURED; 1926 GNUNET_SCHEDULER_shutdown (); 1927 return; 1928 } 1929 TALER_EXCHANGEDB_find_accounts (&process_account_cb, 1930 NULL); 1931 1932 if (0 == test_mode) 1933 { 1934 struct GNUNET_DB_EventHeaderP es = { 1935 .size = htons (sizeof (es)), 1936 .type = htons (TALER_DBEVENT_EXCHANGE_AUDITOR_WAKE_HELPER_WIRE) 1937 }; 1938 1939 eh = TALER_AUDITORDB_event_listen (TALER_ARL_adb, 1940 &es, 1941 GNUNET_TIME_UNIT_FOREVER_REL, 1942 &db_notify, 1943 NULL); 1944 GNUNET_assert (NULL != eh); 1945 } 1946 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1947 begin_transaction ()) 1948 { 1949 GNUNET_break (0); 1950 global_ret = EXIT_FAILURE; 1951 GNUNET_SCHEDULER_shutdown (); 1952 return; 1953 } 1954 } 1955 1956 1957 /** 1958 * The main function of the wire auditing tool. Checks that 1959 * the exchange's records of wire transfers match that of 1960 * the wire gateway. 1961 * 1962 * @param argc number of arguments from the command line 1963 * @param argv command line arguments 1964 * @return 0 ok, 1 on error 1965 */ 1966 int 1967 main (int argc, 1968 char *const *argv) 1969 { 1970 const struct GNUNET_GETOPT_CommandLineOption options[] = { 1971 GNUNET_GETOPT_option_flag ('i', 1972 "internal", 1973 "perform checks only applicable for exchange-internal audits", 1974 &internal_checks), 1975 GNUNET_GETOPT_option_flag ('I', 1976 "ignore-not-found", 1977 "continue, even if the bank account of the exchange was not found", 1978 &ignore_account_404), 1979 GNUNET_GETOPT_option_flag ('t', 1980 "test", 1981 "run in test mode and exit when idle", 1982 &test_mode), 1983 GNUNET_GETOPT_option_timetravel ('T', 1984 "timetravel"), 1985 GNUNET_GETOPT_OPTION_END 1986 }; 1987 enum GNUNET_GenericReturnValue ret; 1988 1989 ret = GNUNET_PROGRAM_run ( 1990 TALER_AUDITOR_project_data (), 1991 argc, 1992 argv, 1993 "taler-helper-auditor-wire-debit", 1994 gettext_noop ( 1995 "Audit exchange database for consistency with the bank's outgoing wire transfers"), 1996 options, 1997 &run, 1998 NULL); 1999 if (GNUNET_SYSERR == ret) 2000 return EXIT_NOTCONFIGURED; 2001 if (GNUNET_NO == ret) 2002 return EXIT_SUCCESS; 2003 return global_ret; 2004 } 2005 2006 2007 /* end of taler-helper-auditor-wire-debit.c */