taler-helper-auditor-wire-debit.c (57701B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2017-2024 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file auditor/taler-helper-auditor-wire-debit.c 18 * @brief audits that wire outgoing transfers match those from an exchange 19 * database. 20 * @author Christian Grothoff 21 * @author Özgür Kesim 22 * 23 * - We check that the outgoing wire transfers match those 24 * given in the 'wire_out' and 'reserve_closures' tables; 25 * any outgoing transfer MUST have a prior justification, 26 * so if one is missing we flag it (and never remove it). 27 * - We check that all wire transfers that should 28 * have been made, were actually made. If any were not made, 29 * we flag those, but may remove those flags if we later 30 * find that the wire transfers were made (wire transfers 31 * could be delayed due to AML/KYC or core-banking issues). 32 */ 33 #include "platform.h" 34 #include <gnunet/gnunet_util_lib.h> 35 #include <gnunet/gnunet_curl_lib.h> 36 #include "auditordb_lib.h" 37 /* WIRE_TRANSFER_OUT's callback typedef also lives in exchangedb_lib.h, so its 38 closure override must be established before that header is included. */ 39 struct WireAccount; 40 #define TALER_EXCHANGEDB_WIRE_TRANSFER_OUT_RESULT_CLOSURE struct WireAccount 41 #include "exchangedb_lib.h" 42 #include "taler/taler_json_lib.h" 43 #include "taler/taler_bank_service.h" 44 #include "taler/taler_signatures.h" 45 #include "report-lib.h" 46 #include "taler/taler_dbevents.h" 47 #include "auditor-database/delete_auditor_closure_lag.h" 48 #include "auditor-database/delete_wire_out_inconsistency_if_matching.h" 49 #include "auditor-database/event_listen.h" 50 #include "auditor-database/get_auditor_progress.h" 51 #include "auditor-database/get_balance.h" 52 #include "auditor-database/insert_auditor_closure_lags.h" 53 #include "auditor-database/insert_auditor_progress.h" 54 #include "auditor-database/insert_balance.h" 55 #include "auditor-database/insert_row_inconsistency.h" 56 #include "auditor-database/insert_row_minor_inconsistencies.h" 57 #include "auditor-database/insert_wire_format_inconsistency.h" 58 #include "auditor-database/insert_wire_out_inconsistency.h" 59 #include "auditor-database/preflight.h" 60 #include "auditor-database/start.h" 61 #include "auditor-database/update_auditor_progress.h" 62 #include "auditor-database/update_balance.h" 63 #include "exchange-database/get_drain_profit.h" 64 #include "exchange-database/preflight.h" 65 #include "exchange-database/rollback.h" 66 #include "exchange-database/select_reserve_closed_above_serial_id.h" 67 #include "exchange-database/select_wire_out_above_serial_id_by_account.h" 68 #include "exchange-database/start_read_only.h" 69 70 71 /** 72 * Maximum number of wire transfers we process per 73 * (database) transaction. 74 */ 75 #define MAX_PER_TRANSACTION 1024 76 77 /** 78 * How much do we allow the bank and the exchange to disagree about 79 * timestamps? Should be sufficiently large to avoid bogus reports from deltas 80 * created by imperfect clock synchronization and network delay. 81 */ 82 #define TIME_TOLERANCE GNUNET_TIME_relative_multiply ( \ 83 GNUNET_TIME_UNIT_MINUTES, \ 84 15) 85 86 87 /** 88 * How long do we try to long-poll for bank wire transfers? 89 */ 90 #define MAX_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 91 GNUNET_TIME_UNIT_HOURS, \ 92 1) 93 94 95 /** 96 * How long do we wait between polling for bank wire transfers at the minimum? 97 */ 98 #define MIN_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 99 GNUNET_TIME_UNIT_MINUTES, \ 100 5) 101 102 103 /** 104 * Run in test mode. Exit when idle instead of 105 * going to sleep and waiting for more work. 106 */ 107 static int test_mode; 108 109 110 /** 111 * Information we keep for each supported account. 112 */ 113 struct WireAccount 114 { 115 /** 116 * Accounts are kept in a DLL. 117 */ 118 struct WireAccount *next; 119 120 /** 121 * Plugins are kept in a DLL. 122 */ 123 struct WireAccount *prev; 124 125 /** 126 * Account details. 127 */ 128 const struct TALER_EXCHANGEDB_AccountInfo *ai; 129 130 /** 131 * Active wire request for the transaction history. 132 */ 133 struct TALER_BANK_DebitHistoryHandle *dhh; 134 135 /** 136 * Task to trigger @e dhh long-polling. 137 */ 138 struct GNUNET_SCHEDULER_Task *dhh_task; 139 140 /** 141 * Time when we expect the current @e dhh long-poll 142 * to finish and we thus could begin another one. 143 */ 144 struct GNUNET_TIME_Absolute dhh_next; 145 146 /** 147 * Progress point for this account. 148 */ 149 uint64_t last_wire_out_serial_id; 150 151 /** 152 * Initial progress point for this account. 153 */ 154 uint64_t start_wire_out_serial_id; 155 156 /** 157 * Where we are in the outbound transaction history. 158 */ 159 uint64_t wire_off_out; 160 161 /** 162 * Label under which we store our pp's reserve_in_serial_id. 163 */ 164 char *label_wire_out_serial_id; 165 166 /** 167 * Label under which we store our wire_off_out. 168 */ 169 char *label_wire_off_out; 170 }; 171 172 173 /** 174 * Information we track for a reserve being closed. 175 */ 176 struct ReserveClosure 177 { 178 /** 179 * Row in the reserves_closed table for this action. 180 */ 181 uint64_t rowid; 182 183 /** 184 * When was the reserve closed? 185 */ 186 struct GNUNET_TIME_Timestamp execution_date; 187 188 /** 189 * Amount transferred (amount remaining minus fee). 190 */ 191 struct TALER_Amount amount; 192 193 /** 194 * Target account where the money was sent. 195 */ 196 struct TALER_FullPayto receiver_account; 197 198 /** 199 * Wire transfer subject used. 200 */ 201 struct TALER_WireTransferIdentifierRawP wtid; 202 }; 203 204 205 /** 206 * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. 207 */ 208 static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; 209 210 /** 211 * Return value from main(). 212 */ 213 static int global_ret; 214 215 /** 216 * State of the current database transaction with 217 * the auditor DB. 218 */ 219 static enum GNUNET_DB_QueryStatus global_qs; 220 221 /** 222 * Map with information about outgoing wire transfers. 223 * Maps hashes of the wire subjects (in binary encoding) 224 * to `struct ReserveOutInfo`s. 225 */ 226 static struct GNUNET_CONTAINER_MultiHashMap *out_map; 227 228 /** 229 * Head of list of wire accounts we still need to look at. 230 */ 231 static struct WireAccount *wa_head; 232 233 /** 234 * Tail of list of wire accounts we still need to look at. 235 */ 236 static struct WireAccount *wa_tail; 237 238 /** 239 * Last reserve_out / wire_out serial IDs seen. 240 */ 241 static TALER_ARL_DEF_PP (wire_reserve_close_id); 242 243 /** 244 * Total amount that was transferred too much from the exchange. 245 */ 246 static TALER_ARL_DEF_AB (total_bad_amount_out_plus); 247 248 /** 249 * Total amount that was transferred too little from the exchange. 250 */ 251 static TALER_ARL_DEF_AB (total_bad_amount_out_minus); 252 253 /** 254 * Total amount of reserve closures which the exchange did not transfer in time. 255 */ 256 static TALER_ARL_DEF_AB (total_closure_amount_lag); 257 258 /** 259 * Total amount affected by duplicate wire transfer 260 * subjects. 261 */ 262 static TALER_ARL_DEF_AB (wire_debit_duplicate_transfer_subject_total); 263 264 /** 265 * Total amount debited to exchange accounts. 266 */ 267 static TALER_ARL_DEF_AB (total_wire_out); 268 269 /** 270 * Total amount of profits drained. 271 */ 272 static TALER_ARL_DEF_AB (total_drained); 273 274 /** 275 * Amount of zero in our currency. 276 */ 277 static struct TALER_Amount zero; 278 279 /** 280 * Handle to the context for interacting with the bank. 281 */ 282 static struct GNUNET_CURL_Context *ctx; 283 284 /** 285 * Scheduler context for running the @e ctx. 286 */ 287 static struct GNUNET_CURL_RescheduleContext *rctx; 288 289 /** 290 * Should we run checks that only work for exchange-internal audits? 291 */ 292 static int internal_checks; 293 294 /** 295 * Should we ignore if the bank does not know our bank 296 * account? 297 */ 298 static int ignore_account_404; 299 300 /** 301 * Database event handler to wake us up again. 302 */ 303 static struct GNUNET_DB_EventHandler *eh; 304 305 /** 306 * The auditors's configuration. 307 */ 308 static const struct GNUNET_CONFIGURATION_Handle *cfg; 309 310 311 /** 312 * Entry in map with wire information we expect to obtain from the 313 * #TALER_ARL_edb later. 314 */ 315 struct WireTransferOutInfo 316 { 317 318 /** 319 * Hash of the wire transfer subject. 320 */ 321 struct GNUNET_HashCode subject_hash; 322 323 /** 324 * Expected details about the wire transfer. 325 */ 326 struct TALER_BANK_DebitDetails details; 327 328 }; 329 330 331 /** 332 * Free entry in #out_map. 333 * 334 * @param cls NULL 335 * @param key unused key 336 * @param value the `struct WireTransferOutInfo` to free 337 * @return #GNUNET_OK 338 */ 339 static enum GNUNET_GenericReturnValue 340 free_roi (void *cls, 341 const struct GNUNET_HashCode *key, 342 void *value) 343 { 344 struct WireTransferOutInfo *roi = value; 345 346 (void) cls; 347 GNUNET_assert (GNUNET_YES == 348 GNUNET_CONTAINER_multihashmap_remove (out_map, 349 key, 350 roi)); 351 GNUNET_free (roi); 352 return GNUNET_OK; 353 } 354 355 356 /** 357 * Free entry in #reserve_closures. 358 * 359 * @param cls NULL 360 * @param key unused key 361 * @param value the `struct ReserveClosure` to free 362 * @return #GNUNET_OK 363 */ 364 static enum GNUNET_GenericReturnValue 365 free_rc (void *cls, 366 const struct GNUNET_HashCode *key, 367 void *value) 368 { 369 struct ReserveClosure *rc = value; 370 371 (void) cls; 372 GNUNET_assert (GNUNET_YES == 373 GNUNET_CONTAINER_multihashmap_remove (reserve_closures, 374 key, 375 rc)); 376 GNUNET_free (rc->receiver_account.full_payto); 377 GNUNET_free (rc); 378 return GNUNET_OK; 379 } 380 381 382 /** 383 * Task run on shutdown. 384 * 385 * @param cls NULL 386 */ 387 static void 388 do_shutdown (void *cls) 389 { 390 struct WireAccount *wa; 391 392 (void) cls; 393 if (NULL != eh) 394 { 395 TALER_AUDITORDB_event_listen_cancel (eh); 396 eh = NULL; 397 } 398 TALER_ARL_done (); 399 if (NULL != reserve_closures) 400 { 401 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 402 &free_rc, 403 NULL); 404 GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); 405 reserve_closures = NULL; 406 } 407 if (NULL != out_map) 408 { 409 GNUNET_CONTAINER_multihashmap_iterate (out_map, 410 &free_roi, 411 NULL); 412 GNUNET_CONTAINER_multihashmap_destroy (out_map); 413 out_map = NULL; 414 } 415 while (NULL != (wa = wa_head)) 416 { 417 if (NULL != wa->dhh_task) 418 { 419 GNUNET_SCHEDULER_cancel (wa->dhh_task); 420 wa->dhh_task = NULL; 421 } 422 if (NULL != wa->dhh) 423 { 424 TALER_BANK_debit_history_cancel (wa->dhh); 425 wa->dhh = NULL; 426 } 427 GNUNET_CONTAINER_DLL_remove (wa_head, 428 wa_tail, 429 wa); 430 GNUNET_free (wa->label_wire_out_serial_id); 431 GNUNET_free (wa->label_wire_off_out); 432 GNUNET_free (wa); 433 } 434 if (NULL != ctx) 435 { 436 GNUNET_CURL_fini (ctx); 437 ctx = NULL; 438 } 439 if (NULL != rctx) 440 { 441 GNUNET_CURL_gnunet_rc_destroy (rctx); 442 rctx = NULL; 443 } 444 TALER_EXCHANGEDB_unload_accounts (); 445 TALER_ARL_cfg = NULL; 446 } 447 448 449 /** 450 * Detect any entries in #reserve_closures that were not yet 451 * observed on the wire transfer side and update the progress 452 * point accordingly. 453 * 454 * @param cls NULL 455 * @param key unused key 456 * @param value the `struct ReserveClosure` to free 457 * @return #GNUNET_OK 458 */ 459 static enum GNUNET_GenericReturnValue 460 check_pending_rc (void *cls, 461 const struct GNUNET_HashCode *key, 462 void *value) 463 { 464 struct ReserveClosure *rc = value; 465 466 (void) cls; 467 (void) key; 468 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 469 "Missing wire transfer for closed reserve with balance %s\n", 470 TALER_amount2s (&rc->amount)); 471 if (! TALER_amount_is_zero (&rc->amount)) 472 { 473 struct TALER_AUDITORDB_ClosureLags cl = { 474 .problem_row_id = rc->rowid, 475 .account = rc->receiver_account, 476 .amount = rc->amount, 477 .deadline = rc->execution_date.abs_time, 478 .wtid = rc->wtid 479 }; 480 enum GNUNET_DB_QueryStatus qs; 481 482 qs = TALER_AUDITORDB_insert_auditor_closure_lags ( 483 TALER_ARL_adb, 484 &cl); 485 if (qs < 0) 486 { 487 global_qs = qs; 488 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 489 return GNUNET_SYSERR; 490 } 491 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_closure_amount_lag), 492 &TALER_ARL_USE_AB (total_closure_amount_lag), 493 &rc->amount); 494 } 495 return GNUNET_OK; 496 } 497 498 499 /** 500 * Compute the key under which a reserve closure for a given 501 * @a receiver_account and @a wtid would be stored. 502 * 503 * @param receiver_account payto://-URI of the account 504 * @param wtid wire transfer identifier used 505 * @param[out] key set to the key 506 */ 507 static void 508 hash_rc (const struct TALER_FullPayto receiver_account, 509 const struct TALER_WireTransferIdentifierRawP *wtid, 510 struct GNUNET_HashCode *key) 511 { 512 struct TALER_NormalizedPayto npto 513 = TALER_payto_normalize (receiver_account); 514 size_t slen = strlen (npto.normalized_payto); 515 char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; 516 517 GNUNET_memcpy (buf, 518 wtid, 519 sizeof (*wtid)); 520 GNUNET_memcpy (&buf[sizeof (*wtid)], 521 npto.normalized_payto, 522 slen); 523 GNUNET_CRYPTO_hash (buf, 524 sizeof (buf), 525 key); 526 GNUNET_free (npto.normalized_payto); 527 } 528 529 530 /** 531 * Start the database transactions and begin the audit. 532 * 533 * @return transaction status code 534 */ 535 static enum GNUNET_DB_QueryStatus 536 begin_transaction (void); 537 538 539 /** 540 * Commit the transaction, checkpointing our progress in the auditor DB. 541 * 542 * @param qs transaction status so far 543 */ 544 static void 545 commit (enum GNUNET_DB_QueryStatus qs) 546 { 547 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 548 &check_pending_rc, 549 NULL); 550 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 551 "Transaction logic ended with status %d\n", 552 qs); 553 TALER_EXCHANGEDB_rollback (TALER_ARL_edb); 554 qs = TALER_AUDITORDB_update_balance ( 555 TALER_ARL_adb, 556 TALER_ARL_SET_AB (total_drained), 557 TALER_ARL_SET_AB (total_wire_out), 558 TALER_ARL_SET_AB (total_bad_amount_out_plus), 559 TALER_ARL_SET_AB (total_bad_amount_out_minus), 560 TALER_ARL_SET_AB (total_closure_amount_lag), 561 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 562 TALER_ARL_SET_AB (total_wire_out), 563 NULL); 564 if (0 > qs) 565 goto handle_db_error; 566 qs = TALER_AUDITORDB_insert_balance ( 567 TALER_ARL_adb, 568 TALER_ARL_SET_AB (total_drained), 569 TALER_ARL_SET_AB (total_wire_out), 570 TALER_ARL_SET_AB (total_bad_amount_out_plus), 571 TALER_ARL_SET_AB (total_bad_amount_out_minus), 572 TALER_ARL_SET_AB (total_closure_amount_lag), 573 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 574 TALER_ARL_SET_AB (total_wire_out), 575 NULL); 576 if (0 > qs) 577 goto handle_db_error; 578 for (struct WireAccount *wa = wa_head; 579 NULL != wa; 580 wa = wa->next) 581 { 582 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 583 "Transaction of account %s ends at %llu/%llu\n", 584 wa->ai->section_name, 585 (unsigned long long) wa->last_wire_out_serial_id, 586 (unsigned long long) wa->wire_off_out); 587 qs = TALER_AUDITORDB_update_auditor_progress ( 588 TALER_ARL_adb, 589 wa->label_wire_out_serial_id, 590 wa->last_wire_out_serial_id, 591 wa->label_wire_off_out, 592 wa->wire_off_out, 593 NULL); 594 if (0 > qs) 595 goto handle_db_error; 596 qs = TALER_AUDITORDB_insert_auditor_progress ( 597 TALER_ARL_adb, 598 wa->label_wire_out_serial_id, 599 wa->last_wire_out_serial_id, 600 wa->label_wire_off_out, 601 wa->wire_off_out, 602 NULL); 603 if (0 > qs) 604 goto handle_db_error; 605 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 606 "Transaction ends at %s=%llu for account `%s'\n", 607 wa->label_wire_out_serial_id, 608 (unsigned long long) wa->last_wire_out_serial_id, 609 wa->ai->section_name); 610 } 611 qs = TALER_AUDITORDB_update_auditor_progress ( 612 TALER_ARL_adb, 613 TALER_ARL_SET_PP (wire_reserve_close_id), 614 NULL); 615 if (0 > qs) 616 goto handle_db_error; 617 qs = TALER_AUDITORDB_insert_auditor_progress ( 618 TALER_ARL_adb, 619 TALER_ARL_SET_PP (wire_reserve_close_id), 620 NULL); 621 if (0 > qs) 622 goto handle_db_error; 623 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 624 "Concluded audit step at %llu\n", 625 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 626 qs = TALER_AUDITORDB_commit (TALER_ARL_adb); 627 if (0 > qs) 628 goto handle_db_error; 629 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 630 "Transaction concluded!\n"); 631 if (1 == test_mode) 632 GNUNET_SCHEDULER_shutdown (); 633 return; 634 handle_db_error: 635 TALER_AUDITORDB_rollback (TALER_ARL_adb); 636 for (unsigned int max_retries = 3; max_retries>0; max_retries--) 637 { 638 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 639 break; 640 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 641 "Serialization issue, trying again\n"); 642 qs = begin_transaction (); 643 } 644 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 645 "Hard database error, terminating\n"); 646 GNUNET_SCHEDULER_shutdown (); 647 } 648 649 650 /** 651 * Check that @a want is within #TIME_TOLERANCE of @a have. 652 * Otherwise report an inconsistency in row @a rowid of @a table. 653 * 654 * @param table where is the inconsistency (if any) 655 * @param rowid what is the row 656 * @param want what is the expected time 657 * @param have what is the time we got 658 * @return true on success, false to abort 659 */ 660 static bool 661 check_time_difference (const char *table, 662 uint64_t rowid, 663 struct GNUNET_TIME_Timestamp want, 664 struct GNUNET_TIME_Timestamp have) 665 { 666 struct GNUNET_TIME_Relative delta; 667 char *details; 668 669 if (GNUNET_TIME_timestamp_cmp (have, >, want)) 670 delta = GNUNET_TIME_absolute_get_difference (want.abs_time, 671 have.abs_time); 672 else 673 delta = GNUNET_TIME_absolute_get_difference (have.abs_time, 674 want.abs_time); 675 if (GNUNET_TIME_relative_cmp (delta, 676 <=, 677 TIME_TOLERANCE)) 678 return true; 679 680 GNUNET_asprintf (&details, 681 "execution date mismatch (%s)", 682 GNUNET_TIME_relative2s (delta, 683 true)); 684 { 685 struct TALER_AUDITORDB_RowMinorInconsistencies rmi = { 686 .row_table = (char *) table, 687 .problem_row = rowid, 688 .diagnostic = details 689 }; 690 enum GNUNET_DB_QueryStatus qs; 691 692 qs = TALER_AUDITORDB_insert_row_minor_inconsistencies ( 693 TALER_ARL_adb, 694 &rmi); 695 696 if (qs < 0) 697 { 698 global_qs = qs; 699 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 700 GNUNET_free (details); 701 return false; 702 } 703 } 704 GNUNET_free (details); 705 return true; 706 } 707 708 709 /** 710 * Closure for #check_rc_matches 711 */ 712 struct CheckMatchContext 713 { 714 715 /** 716 * Reserve operation looking for a match 717 */ 718 const struct WireTransferOutInfo *roi; 719 720 /** 721 * Set to true if we found a match. 722 */ 723 bool found; 724 }; 725 726 727 /** 728 * Check if any of the reserve closures match the given wire transfer. 729 * 730 * @param[in,out] cls a `struct CheckMatchContext` 731 * @param key key of @a value in #reserve_closures 732 * @param value a `struct ReserveClosure` 733 */ 734 static enum GNUNET_GenericReturnValue 735 check_rc_matches (void *cls, 736 const struct GNUNET_HashCode *key, 737 void *value) 738 { 739 struct CheckMatchContext *cmx = cls; 740 struct ReserveClosure *rc = value; 741 742 if ((0 == GNUNET_memcmp (&cmx->roi->details.wtid, 743 &rc->wtid)) && 744 (0 == TALER_full_payto_cmp (rc->receiver_account, 745 cmx->roi->details.credit_account_uri)) && 746 (0 == TALER_amount_cmp (&rc->amount, 747 &cmx->roi->details.amount))) 748 { 749 if (! check_time_difference ("reserves_closures", 750 rc->rowid, 751 rc->execution_date, 752 cmx->roi->details.execution_date)) 753 { 754 free_rc (NULL, 755 key, 756 rc); 757 return GNUNET_SYSERR; 758 } 759 cmx->found = true; 760 free_rc (NULL, 761 key, 762 rc); 763 return GNUNET_NO; 764 } 765 return GNUNET_OK; 766 } 767 768 769 /** 770 * Maximum required length for make_missing_diag(). 771 */ 772 #define MAX_DIAG_LEN 128 773 774 /** 775 * Make diagnostic string for missing wire transfer. 776 * 777 * @param[out] diag where to write the diagnostic string 778 * @param wtid wire transfer ID to include 779 */ 780 static void 781 make_missing_diag (char diag[MAX_DIAG_LEN], 782 const struct TALER_WireTransferIdentifierRawP *wtid) 783 { 784 char *wtid_s; 785 786 wtid_s = GNUNET_STRINGS_data_to_string_alloc (wtid, 787 sizeof (*wtid)); 788 GNUNET_snprintf (diag, 789 MAX_DIAG_LEN, 790 "expected outgoing wire transfer %s missing", 791 wtid_s); 792 GNUNET_free (wtid_s); 793 } 794 795 796 /** 797 * Check if an existing report on a missing wire 798 * out operation justified the @a roi. If so, 799 * clear the existing report. 800 * 801 * @param roi reserve out operation to check 802 * @return #GNUNET_YES if @a roi was justified by a previous report, 803 * #GNUNET_NO of @a roi was not justified by a previous missing report 804 * #GNUNET_SYSERR on database trouble 805 */ 806 static enum GNUNET_GenericReturnValue 807 check_reported_inconsistency (struct WireTransferOutInfo *roi) 808 { 809 char diag[MAX_DIAG_LEN]; 810 struct TALER_AUDITORDB_WireOutInconsistency woi = { 811 .wire_out_row_id = roi->details.serial_id, 812 .destination_account = roi->details.credit_account_uri, 813 .diagnostic = diag, 814 .expected = roi->details.amount, 815 .claimed = zero, 816 }; 817 enum GNUNET_DB_QueryStatus qs; 818 819 make_missing_diag (diag, 820 &roi->details.wtid); 821 qs = TALER_AUDITORDB_delete_wire_out_inconsistency_if_matching ( 822 TALER_ARL_adb, 823 &woi); 824 if (qs < 0) 825 { 826 global_qs = qs; 827 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 828 return GNUNET_SYSERR; 829 } 830 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 831 { 832 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 833 "Deletion of wire out inconsistency %llu (%s, %s, %s) failed: not reported missing!\n", 834 (unsigned long long) roi->details.serial_id, 835 roi->details.credit_account_uri.full_payto, 836 diag, 837 TALER_amount2s (&roi->details.amount)); 838 return GNUNET_NO; 839 } 840 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 841 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 842 &roi->details.amount); 843 return GNUNET_YES; 844 } 845 846 847 /** 848 * Check if a profit drain operation justified the @a roi 849 * 850 * @param roi reserve out operation to check 851 * @return #GNUNET_YES if @a roi was justified by a profit drain, 852 * #GNUNET_NO of @a roi was not justified by a proft drain 853 * #GNUNET_SYSERR on database trouble 854 */ 855 static enum GNUNET_GenericReturnValue 856 check_profit_drain (struct WireTransferOutInfo *roi) 857 { 858 enum GNUNET_DB_QueryStatus qs; 859 uint64_t serial; 860 char *account_section; 861 struct TALER_FullPayto payto_uri; 862 struct GNUNET_TIME_Timestamp request_timestamp; 863 struct TALER_Amount amount; 864 struct TALER_MasterSignatureP master_sig; 865 866 qs = TALER_EXCHANGEDB_get_drain_profit ( 867 TALER_ARL_edb, 868 &roi->details.wtid, 869 &serial, 870 &account_section, 871 &payto_uri, 872 &request_timestamp, 873 &amount, 874 &master_sig); 875 switch (qs) 876 { 877 case GNUNET_DB_STATUS_HARD_ERROR: 878 GNUNET_break (0); 879 global_ret = EXIT_FAILURE; 880 GNUNET_SCHEDULER_shutdown (); 881 return GNUNET_SYSERR; 882 case GNUNET_DB_STATUS_SOFT_ERROR: 883 /* should fail on commit later ... */ 884 GNUNET_break (0); 885 return GNUNET_SYSERR; 886 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 887 /* not a profit drain */ 888 return GNUNET_NO; 889 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 890 break; 891 } 892 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 893 "Profit drain of %s to %s found!\n", 894 TALER_amount2s (&amount), 895 payto_uri.full_payto); 896 if (GNUNET_OK != 897 TALER_exchange_offline_profit_drain_verify ( 898 &roi->details.wtid, 899 request_timestamp, 900 &amount, 901 account_section, 902 payto_uri, 903 &TALER_ARL_master_pub, 904 &master_sig)) 905 { 906 struct TALER_AUDITORDB_RowInconsistency ri = { 907 .row_id = roi->details.serial_id, 908 .row_table = (char *) "profit_drains", 909 .diagnostic = (char *) "invalid signature" 910 }; 911 912 GNUNET_break (0); 913 qs = TALER_AUDITORDB_insert_row_inconsistency ( 914 TALER_ARL_adb, 915 &ri); 916 GNUNET_free (payto_uri.full_payto); 917 GNUNET_free (account_section); 918 if (qs < 0) 919 { 920 global_qs = qs; 921 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 922 return GNUNET_SYSERR; 923 } 924 return GNUNET_NO; 925 } 926 GNUNET_free (account_section); 927 928 { 929 if (0 != 930 TALER_full_payto_normalize_and_cmp (payto_uri, 931 roi->details.credit_account_uri)) 932 { 933 struct TALER_AUDITORDB_WireOutInconsistency woi = { 934 .wire_out_row_id = serial, 935 .destination_account = roi->details.credit_account_uri, 936 .diagnostic = (char *) "profit drain wired to invalid account", 937 .expected = roi->details.amount, 938 .claimed = zero, 939 }; 940 941 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 942 TALER_ARL_adb, 943 &woi); 944 if (qs < 0) 945 { 946 global_qs = qs; 947 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 948 GNUNET_free (payto_uri.full_payto); 949 return GNUNET_SYSERR; 950 } 951 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 952 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 953 &amount); 954 GNUNET_free (payto_uri.full_payto); 955 return GNUNET_YES; /* justified, kind-of */ 956 } 957 } 958 GNUNET_free (payto_uri.full_payto); 959 if (0 != 960 TALER_amount_cmp (&amount, 961 &roi->details.amount)) 962 { 963 struct TALER_AUDITORDB_WireOutInconsistency woi = { 964 .wire_out_row_id = roi->details.serial_id, 965 .destination_account = roi->details.credit_account_uri, 966 .diagnostic = (char *) "incorrect amount drained to correct account", 967 .expected = roi->details.amount, 968 .claimed = amount, 969 }; 970 971 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 972 TALER_ARL_adb, 973 &woi); 974 if (qs < 0) 975 { 976 global_qs = qs; 977 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 978 return GNUNET_SYSERR; 979 } 980 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 981 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 982 &roi->details.amount); 983 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 984 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 985 &amount); 986 return GNUNET_YES; /* justified, kind-of */ 987 } 988 /* profit drain was correct */ 989 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_drained), 990 &TALER_ARL_USE_AB (total_drained), 991 &amount); 992 return GNUNET_YES; 993 } 994 995 996 /** 997 * Check whether the given transfer was justified because it was 998 * actually previously reported as missing. 999 * 1000 * @param roi the reserve out operation to check 1001 * @return #GNUNET_OK on success, #GNUNET_NO if there was no 1002 * matching lag, #GNUNET_SYSERR on database trouble 1003 */ 1004 static enum GNUNET_GenericReturnValue 1005 check_closure_lag (const struct WireTransferOutInfo *roi) 1006 { 1007 enum GNUNET_DB_QueryStatus qs; 1008 1009 qs = TALER_AUDITORDB_delete_auditor_closure_lag ( 1010 TALER_ARL_adb, 1011 &roi->details.amount, 1012 &roi->details.wtid, 1013 roi->details.credit_account_uri); 1014 if (qs < 0) 1015 { 1016 global_qs = qs; 1017 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1018 return GNUNET_SYSERR; 1019 } 1020 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1021 return GNUNET_NO; 1022 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1023 "Cleared closure lag: found justification\n"); 1024 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_closure_amount_lag), 1025 &TALER_ARL_USE_AB (total_closure_amount_lag), 1026 &roi->details.amount); 1027 return GNUNET_YES; /* found! */ 1028 } 1029 1030 1031 /** 1032 * Check whether the given transfer was justified by a reserve closure or 1033 * profit drain. If not, complain that we failed to match an entry from 1034 * #out_map. This means a wire transfer was made without proper 1035 * justification. 1036 * 1037 * @param cls a `struct WireAccount` 1038 * @param key unused key 1039 * @param value the `struct WireTransferOutInfo` to report 1040 * @return #GNUNET_OK on success 1041 */ 1042 static enum GNUNET_GenericReturnValue 1043 complain_out_not_found (void *cls, 1044 const struct GNUNET_HashCode *key, 1045 void *value) 1046 { 1047 // struct WireAccount *wa = cls; 1048 struct WireTransferOutInfo *roi = value; 1049 struct GNUNET_HashCode rkey; 1050 struct CheckMatchContext cmx = { 1051 .roi = roi, 1052 .found = false 1053 }; 1054 enum GNUNET_GenericReturnValue ret; 1055 1056 (void) cls; 1057 (void) key; 1058 hash_rc (roi->details.credit_account_uri, 1059 &roi->details.wtid, 1060 &rkey); 1061 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1062 "Checking for reserve closure %s benefiting %s\n", 1063 GNUNET_h2s (&rkey), 1064 roi->details.credit_account_uri.full_payto); 1065 GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, 1066 &rkey, 1067 &check_rc_matches, 1068 &cmx); 1069 if (cmx.found) 1070 return GNUNET_OK; 1071 ret = check_reported_inconsistency (roi); 1072 if (GNUNET_NO != ret) 1073 return ret; 1074 ret = check_profit_drain (roi); 1075 if (GNUNET_NO != ret) 1076 return ret; 1077 ret = check_closure_lag (roi); 1078 if (GNUNET_NO != ret) 1079 return ret; 1080 1081 { 1082 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1083 .destination_account = roi->details.credit_account_uri, 1084 .diagnostic = (char *) "missing justification for outgoing wire transfer", 1085 .wire_out_row_id = roi->details.serial_id, 1086 .expected = zero, 1087 .claimed = roi->details.amount 1088 }; 1089 enum GNUNET_DB_QueryStatus qs; 1090 1091 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1092 TALER_ARL_adb, 1093 &woi); 1094 if (qs < 0) 1095 { 1096 global_qs = qs; 1097 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1098 return GNUNET_SYSERR; 1099 } 1100 } 1101 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1102 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1103 &roi->details.amount); 1104 return GNUNET_OK; 1105 } 1106 1107 1108 /** 1109 * Function called with details about outgoing wire transfers 1110 * as claimed by the exchange DB. 1111 * 1112 * @param wa a `struct WireAccount` 1113 * @param rowid unique serial ID in wire_out table 1114 * @param date timestamp of the transfer (roughly) 1115 * @param wtid wire transfer subject 1116 * @param payto_uri wire transfer details of the receiver 1117 * @param amount amount that was wired 1118 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1119 */ 1120 static enum GNUNET_GenericReturnValue 1121 wire_out_cb ( 1122 struct WireAccount *wa, 1123 uint64_t rowid, 1124 struct GNUNET_TIME_Timestamp date, 1125 const struct TALER_WireTransferIdentifierRawP *wtid, 1126 const struct TALER_FullPayto payto_uri, 1127 const struct TALER_Amount *amount) 1128 { 1129 struct GNUNET_HashCode key; 1130 struct WireTransferOutInfo *roi; 1131 1132 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1133 "Exchange wire OUT #%llu at %s of %s with WTID %s\n", 1134 (unsigned long long) rowid, 1135 GNUNET_TIME_timestamp2s (date), 1136 TALER_amount2s (amount), 1137 TALER_B2S (wtid)); 1138 wa->last_wire_out_serial_id = rowid + 1; 1139 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_wire_out), 1140 &TALER_ARL_USE_AB (total_wire_out), 1141 amount); 1142 GNUNET_CRYPTO_hash (wtid, 1143 sizeof (*wtid), 1144 &key); 1145 roi = GNUNET_CONTAINER_multihashmap_get (out_map, 1146 &key); 1147 if (NULL == roi) 1148 { 1149 /* Wire transfer was not made (yet) at all (but would have been 1150 justified), so the entire amount is missing / still to be done. This 1151 is moderately harmless, it might just be that the 1152 taler-exchange-transfer tool or bank has not yet fully caught up with 1153 the transfers it should do. 1154 May be cleared later by check_reported_inconsistency() */ 1155 char diag[MAX_DIAG_LEN]; 1156 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1157 .destination_account = payto_uri, 1158 .diagnostic = diag, 1159 .wire_out_row_id = rowid, 1160 .expected = *amount, 1161 .claimed = zero, 1162 }; 1163 enum GNUNET_DB_QueryStatus qs; 1164 1165 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1166 "Wire out for row %llu still missing\n", 1167 (unsigned long long) rowid); 1168 make_missing_diag (diag, 1169 wtid); 1170 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1171 TALER_ARL_adb, 1172 &woi); 1173 if (qs < 0) 1174 { 1175 global_qs = qs; 1176 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1177 return GNUNET_SYSERR; 1178 } 1179 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1180 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1181 amount); 1182 return GNUNET_OK; 1183 } 1184 1185 if (0 != TALER_full_payto_normalize_and_cmp (payto_uri, 1186 roi->details.credit_account_uri)) 1187 { 1188 /* Destination bank account is wrong in actual wire transfer, so 1189 we should count the wire transfer as entirely spurious, and 1190 additionally consider the justified wire transfer as missing. */ 1191 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1192 .wire_out_row_id = rowid, 1193 .destination_account = payto_uri, 1194 .diagnostic = (char *) "receiver account mismatch", 1195 .expected = *amount, 1196 .claimed = roi->details.amount, 1197 }; 1198 enum GNUNET_DB_QueryStatus qs; 1199 1200 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1201 TALER_ARL_adb, 1202 &woi); 1203 if (qs < 0) 1204 { 1205 global_qs = qs; 1206 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1207 return GNUNET_SYSERR; 1208 } 1209 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1210 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1211 &roi->details.amount); 1212 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1213 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1214 amount); 1215 GNUNET_assert (GNUNET_OK == 1216 free_roi (NULL, 1217 &key, 1218 roi)); 1219 return GNUNET_OK; 1220 } 1221 1222 if (0 != TALER_amount_cmp (&roi->details.amount, 1223 amount)) 1224 { 1225 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1226 .destination_account = payto_uri, 1227 .diagnostic = (char *) "wire amount does not match", 1228 .wire_out_row_id = rowid, 1229 .expected = *amount, 1230 .claimed = roi->details.amount, 1231 }; 1232 enum GNUNET_DB_QueryStatus qs; 1233 1234 qs = TALER_AUDITORDB_insert_wire_out_inconsistency ( 1235 TALER_ARL_adb, 1236 &woi); 1237 if (qs < 0) 1238 { 1239 global_qs = qs; 1240 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1241 return GNUNET_SYSERR; 1242 } 1243 if (0 < TALER_amount_cmp (amount, 1244 &roi->details.amount)) 1245 { 1246 /* amount > roi->details.amount: wire transfer was smaller than it should have been */ 1247 struct TALER_Amount delta; 1248 1249 TALER_ARL_amount_subtract (&delta, 1250 amount, 1251 &roi->details.amount); 1252 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1253 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1254 &delta); 1255 } 1256 else 1257 { 1258 /* roi->details.amount < amount: wire transfer was larger than it should have been */ 1259 struct TALER_Amount delta; 1260 1261 TALER_ARL_amount_subtract (&delta, 1262 &roi->details.amount, 1263 amount); 1264 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1265 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1266 &delta); 1267 } 1268 GNUNET_assert (GNUNET_OK == 1269 free_roi (NULL, 1270 &key, 1271 roi)); 1272 return GNUNET_OK; 1273 } 1274 1275 { 1276 enum GNUNET_GenericReturnValue ret; 1277 1278 if (! check_time_difference ("wire_out", 1279 rowid, 1280 date, 1281 roi->details.execution_date)) 1282 { 1283 /* We had a database error, fail */ 1284 ret = GNUNET_SYSERR; 1285 } 1286 else 1287 { 1288 ret = GNUNET_OK; 1289 } 1290 GNUNET_assert (GNUNET_OK == 1291 free_roi (NULL, 1292 &key, 1293 roi)); 1294 return ret; 1295 } 1296 } 1297 1298 1299 /** 1300 * Main function for processing 'debit' data. We start by going over 1301 * the DEBIT transactions this time, and then verify that all of them are 1302 * justified by reserve closures, profit drains or regular outgoing 1303 * wire transfers from aggregated deposits. 1304 * 1305 * @param[in,out] wa wire account list to process 1306 */ 1307 static void 1308 process_debits (struct WireAccount *wa); 1309 1310 1311 /** 1312 * Go over the "wire_out" table of the exchange and 1313 * verify that all wire outs are in that table. 1314 * 1315 * @param[in,out] wa wire account we are processing 1316 */ 1317 static void 1318 check_exchange_wire_out (struct WireAccount *wa) 1319 { 1320 enum GNUNET_DB_QueryStatus qs; 1321 1322 GNUNET_assert (NULL == wa->dhh); 1323 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1324 "Analyzing exchange's wire OUT table for account `%s'\n", 1325 wa->ai->section_name); 1326 qs = TALER_TALER_EXCHANGEDB_select_wire_out_above_serial_id_by_account ( 1327 TALER_ARL_edb, 1328 wa->ai->section_name, 1329 wa->last_wire_out_serial_id, 1330 &wire_out_cb, 1331 wa); 1332 if (0 > qs) 1333 { 1334 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1335 global_ret = EXIT_FAILURE; 1336 GNUNET_SCHEDULER_shutdown (); 1337 return; 1338 } 1339 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1340 &complain_out_not_found, 1341 wa); 1342 /* clean up */ 1343 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1344 &free_roi, 1345 NULL); 1346 process_debits (wa->next); 1347 } 1348 1349 1350 /** 1351 * This function is called for all transactions that 1352 * are debited from the exchange's account (outgoing 1353 * transactions). 1354 * 1355 * @param cls `struct WireAccount` with current wire account to process 1356 * @param dhr HTTP response details 1357 */ 1358 static void 1359 history_debit_cb ( 1360 void *cls, 1361 const struct TALER_BANK_DebitHistoryResponse *dhr); 1362 1363 1364 /** 1365 * Task scheduled to begin long-polling on the 1366 * bank transfer. 1367 * 1368 * @param cls a `struct WireAccount *` 1369 */ 1370 static void 1371 dh_long_poll (void *cls) 1372 { 1373 struct WireAccount *wa = cls; 1374 1375 wa->dhh_task = NULL; 1376 wa->dhh_next 1377 = GNUNET_TIME_relative_to_absolute (MIN_LONGPOLL_DELAY); 1378 GNUNET_assert (NULL == wa->dhh); 1379 wa->dhh = TALER_BANK_debit_history ( 1380 ctx, 1381 wa->ai->auth, 1382 wa->wire_off_out, 1383 MAX_PER_TRANSACTION, 1384 MAX_LONGPOLL_DELAY, 1385 &history_debit_cb, 1386 wa); 1387 if (NULL == wa->dhh) 1388 { 1389 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1390 "Failed to start long-polling for bank transaction history for `%s'\n", 1391 wa->ai->section_name); 1392 global_ret = EXIT_FAILURE; 1393 GNUNET_SCHEDULER_shutdown (); 1394 return; 1395 } 1396 } 1397 1398 1399 static void 1400 history_debit_cb ( 1401 void *cls, 1402 const struct TALER_BANK_DebitHistoryResponse *dhr) 1403 { 1404 struct WireAccount *wa = cls; 1405 struct WireTransferOutInfo *roi; 1406 size_t slen; 1407 1408 wa->dhh = NULL; 1409 if ( (MHD_HTTP_OK == dhr->http_status) && 1410 (0 != dhr->details.ok.details_length) ) 1411 { 1412 /* As we got results, we go again *immediately* */ 1413 wa->dhh_next = GNUNET_TIME_UNIT_ZERO_ABS; 1414 } 1415 GNUNET_assert (NULL == wa->dhh_task); 1416 wa->dhh_task 1417 = GNUNET_SCHEDULER_add_at (wa->dhh_next, 1418 &dh_long_poll, 1419 wa); 1420 switch (dhr->http_status) 1421 { 1422 case MHD_HTTP_OK: 1423 for (unsigned int i = 0; i < dhr->details.ok.details_length; i++) 1424 { 1425 const struct TALER_BANK_DebitDetails *dd 1426 = &dhr->details.ok.details[i]; 1427 1428 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1429 "Analyzing bank DEBIT #%llu at %s of %s with WTID %s\n", 1430 (unsigned long long) dd->serial_id, 1431 GNUNET_TIME_timestamp2s (dd->execution_date), 1432 TALER_amount2s (&dd->amount), 1433 TALER_B2S (&dd->wtid)); 1434 wa->wire_off_out = dd->serial_id + 1; 1435 slen = strlen (dd->credit_account_uri.full_payto) + 1; 1436 roi = GNUNET_malloc (sizeof (struct WireTransferOutInfo) 1437 + slen); 1438 GNUNET_CRYPTO_hash (&dd->wtid, 1439 sizeof (dd->wtid), 1440 &roi->subject_hash); 1441 roi->details = *dd; 1442 roi->details.credit_account_uri.full_payto 1443 = (char *) &roi[1]; 1444 GNUNET_memcpy (&roi[1], 1445 dd->credit_account_uri.full_payto, 1446 slen); 1447 if (GNUNET_OK != 1448 GNUNET_CONTAINER_multihashmap_put (out_map, 1449 &roi->subject_hash, 1450 roi, 1451 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) 1452 { 1453 struct TALER_AUDITORDB_WireFormatInconsistency wfi = { 1454 .amount = dd->amount, 1455 .wire_offset = dd->serial_id, 1456 .diagnostic = (char *) "duplicate outgoing wire transfer subject" 1457 }; 1458 enum GNUNET_DB_QueryStatus qs; 1459 1460 qs = TALER_AUDITORDB_insert_wire_format_inconsistency ( 1461 TALER_ARL_adb, 1462 &wfi); 1463 if (qs < 0) 1464 { 1465 global_qs = qs; 1466 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1467 commit (qs); 1468 return; 1469 } 1470 TALER_ARL_amount_add (&TALER_ARL_USE_AB ( 1471 wire_debit_duplicate_transfer_subject_total), 1472 &TALER_ARL_USE_AB ( 1473 wire_debit_duplicate_transfer_subject_total), 1474 &dd->amount); 1475 } 1476 } 1477 check_exchange_wire_out (wa); 1478 return; 1479 case MHD_HTTP_NO_CONTENT: 1480 check_exchange_wire_out (wa); 1481 return; 1482 case MHD_HTTP_NOT_FOUND: 1483 if (ignore_account_404) 1484 { 1485 check_exchange_wire_out (wa); 1486 return; 1487 } 1488 break; 1489 default: 1490 break; 1491 } 1492 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1493 "Error fetching debit history of account %s: %u/%u!\n", 1494 wa->ai->section_name, 1495 dhr->http_status, 1496 (unsigned int) dhr->ec); 1497 commit (GNUNET_DB_STATUS_HARD_ERROR); 1498 global_ret = EXIT_FAILURE; 1499 GNUNET_SCHEDULER_shutdown (); 1500 return; 1501 } 1502 1503 1504 static void 1505 process_debits (struct WireAccount *wa) 1506 { 1507 /* skip accounts where DEBIT is not enabled */ 1508 while ( (NULL != wa) && 1509 (! wa->ai->debit_enabled) ) 1510 wa = wa->next; 1511 if (NULL == wa) 1512 { 1513 /* end of iteration */ 1514 commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); 1515 return; 1516 } 1517 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1518 "Checking bank DEBIT records of account `%s'\n", 1519 wa->ai->section_name); 1520 if ( (NULL == wa->dhh) && 1521 (NULL == wa->dhh_task) ) 1522 { 1523 wa->dhh = TALER_BANK_debit_history ( 1524 ctx, 1525 wa->ai->auth, 1526 wa->wire_off_out, 1527 MAX_PER_TRANSACTION, 1528 GNUNET_TIME_UNIT_ZERO, 1529 &history_debit_cb, 1530 wa); 1531 if (NULL == wa->dhh) 1532 { 1533 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1534 "Failed to obtain bank transaction history for `%s'\n", 1535 wa->ai->section_name); 1536 commit (GNUNET_DB_STATUS_HARD_ERROR); 1537 global_ret = EXIT_FAILURE; 1538 GNUNET_SCHEDULER_shutdown (); 1539 return; 1540 } 1541 } 1542 } 1543 1544 1545 /** 1546 * Function called about reserve closing operations the aggregator triggered. 1547 * 1548 * @param cls closure; NULL 1549 * @param rowid row identifier used to uniquely identify the reserve closing operation 1550 * @param execution_date when did we execute the close operation 1551 * @param amount_with_fee how much did we debit the reserve 1552 * @param closing_fee how much did we charge for closing the reserve 1553 * @param reserve_pub public key of the reserve 1554 * @param receiver_account where did we send the funds, in payto://-format 1555 * @param wtid identifier used for the wire transfer 1556 * @param close_request_row which close request triggered the operation? 1557 * 0 if it was a timeout (not used) 1558 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1559 */ 1560 static enum GNUNET_GenericReturnValue 1561 reserve_closed_cb ( 1562 void *cls, 1563 uint64_t rowid, 1564 struct GNUNET_TIME_Timestamp execution_date, 1565 const struct TALER_Amount *amount_with_fee, 1566 const struct TALER_Amount *closing_fee, 1567 const struct TALER_ReservePublicKeyP *reserve_pub, 1568 const struct TALER_FullPayto receiver_account, 1569 const struct TALER_WireTransferIdentifierRawP *wtid, 1570 uint64_t close_request_row) 1571 { 1572 struct ReserveClosure *rc; 1573 struct GNUNET_HashCode key; 1574 1575 (void) cls; 1576 (void) close_request_row; 1577 GNUNET_assert (TALER_ARL_USE_PP (wire_reserve_close_id) <= rowid); 1578 TALER_ARL_USE_PP (wire_reserve_close_id) = rowid + 1; 1579 rc = GNUNET_new (struct ReserveClosure); 1580 if (TALER_ARL_SR_INVALID_NEGATIVE == 1581 TALER_ARL_amount_subtract_neg (&rc->amount, 1582 amount_with_fee, 1583 closing_fee)) 1584 { 1585 struct TALER_AUDITORDB_RowInconsistency ri = { 1586 .row_id = rowid, 1587 .row_table 1588 = (char *) "reserves_closures", 1589 .diagnostic 1590 = (char *) "closing fee above reserve balance (and closed anyway)" 1591 }; 1592 enum GNUNET_DB_QueryStatus qs; 1593 1594 qs = TALER_AUDITORDB_insert_row_inconsistency ( 1595 TALER_ARL_adb, 1596 &ri); 1597 if (qs < 0) 1598 { 1599 global_qs = qs; 1600 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1601 return GNUNET_OK; 1602 } 1603 GNUNET_free (rc); 1604 return GNUNET_OK; 1605 } 1606 rc->receiver_account.full_payto 1607 = GNUNET_strdup (receiver_account.full_payto); 1608 rc->wtid = *wtid; 1609 rc->execution_date = execution_date; 1610 rc->rowid = rowid; 1611 hash_rc (rc->receiver_account, 1612 wtid, 1613 &key); 1614 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1615 "Discovered reserve closure %llu (%s) over %s benefiting %s\n", 1616 (unsigned long long) rowid, 1617 GNUNET_h2s (&key), 1618 TALER_amount2s (amount_with_fee), 1619 receiver_account.full_payto); 1620 (void) GNUNET_CONTAINER_multihashmap_put ( 1621 reserve_closures, 1622 &key, 1623 rc, 1624 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); 1625 return GNUNET_OK; 1626 } 1627 1628 1629 /** 1630 * Start the database transactions and begin the audit. 1631 * 1632 * @return transaction status code 1633 */ 1634 static enum GNUNET_DB_QueryStatus 1635 begin_transaction (void) 1636 { 1637 enum GNUNET_DB_QueryStatus qs; 1638 1639 if (GNUNET_SYSERR == 1640 TALER_EXCHANGEDB_preflight (TALER_ARL_edb)) 1641 { 1642 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1643 "Failed to initialize exchange database connection.\n"); 1644 return GNUNET_DB_STATUS_HARD_ERROR; 1645 } 1646 if (GNUNET_SYSERR == 1647 TALER_AUDITORDB_preflight (TALER_ARL_adb)) 1648 { 1649 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1650 "Failed to initialize auditor database session.\n"); 1651 return GNUNET_DB_STATUS_HARD_ERROR; 1652 } 1653 global_qs = GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1654 if (GNUNET_OK != 1655 TALER_AUDITORDB_start (TALER_ARL_adb, 1656 "auditor-wire-debit")) 1657 { 1658 GNUNET_break (0); 1659 return GNUNET_DB_STATUS_HARD_ERROR; 1660 } 1661 if (GNUNET_OK != 1662 TALER_TALER_EXCHANGEDB_start_read_only (TALER_ARL_edb, 1663 "wire debit auditor")) 1664 { 1665 GNUNET_break (0); 1666 return GNUNET_DB_STATUS_HARD_ERROR; 1667 } 1668 qs = TALER_AUDITORDB_get_balance ( 1669 TALER_ARL_adb, 1670 TALER_ARL_GET_AB (total_drained), 1671 TALER_ARL_GET_AB (total_wire_out), 1672 TALER_ARL_GET_AB (total_bad_amount_out_plus), 1673 TALER_ARL_GET_AB (total_bad_amount_out_minus), 1674 TALER_ARL_GET_AB (total_closure_amount_lag), 1675 TALER_ARL_GET_AB (wire_debit_duplicate_transfer_subject_total), 1676 TALER_ARL_GET_AB (total_wire_out), 1677 NULL); 1678 switch (qs) 1679 { 1680 case GNUNET_DB_STATUS_HARD_ERROR: 1681 GNUNET_break (0); 1682 return qs; 1683 case GNUNET_DB_STATUS_SOFT_ERROR: 1684 GNUNET_break (0); 1685 return qs; 1686 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 1687 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 1688 break; 1689 } 1690 for (struct WireAccount *wa = wa_head; 1691 NULL != wa; 1692 wa = wa->next) 1693 { 1694 GNUNET_asprintf (&wa->label_wire_out_serial_id, 1695 "wire-%s-%s", 1696 wa->ai->section_name, 1697 "wire_out_serial_id"); 1698 GNUNET_asprintf (&wa->label_wire_off_out, 1699 "wire-%s-%s", 1700 wa->ai->section_name, 1701 "wire_off_out"); 1702 qs = TALER_AUDITORDB_get_auditor_progress ( 1703 TALER_ARL_adb, 1704 wa->label_wire_out_serial_id, 1705 &wa->last_wire_out_serial_id, 1706 wa->label_wire_off_out, 1707 &wa->wire_off_out, 1708 NULL); 1709 if (0 > qs) 1710 { 1711 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1712 return qs; 1713 } 1714 GNUNET_assert (2 == qs); 1715 wa->start_wire_out_serial_id = wa->last_wire_out_serial_id; 1716 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1717 "Resuming account %s debit audit at %llu/%llu\n", 1718 wa->ai->section_name, 1719 (unsigned long long) wa->last_wire_out_serial_id, 1720 (unsigned long long) wa->wire_off_out); 1721 } 1722 qs = TALER_AUDITORDB_get_auditor_progress ( 1723 TALER_ARL_adb, 1724 TALER_ARL_GET_PP (wire_reserve_close_id), 1725 NULL); 1726 if (0 > qs) 1727 { 1728 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1729 return qs; 1730 } 1731 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1732 { 1733 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, 1734 "First analysis of with wire auditor, starting audit from scratch\n"); 1735 } 1736 else 1737 { 1738 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1739 "Resuming wire debit audit at %llu\n", 1740 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1741 } 1742 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1743 "Iterating over reserve closures from %llu\n", 1744 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1745 qs = TALER_EXCHANGEDB_select_reserve_closed_above_serial_id ( 1746 TALER_ARL_edb, 1747 TALER_ARL_USE_PP (wire_reserve_close_id), 1748 &reserve_closed_cb, 1749 NULL); 1750 if (0 > qs) 1751 { 1752 GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); 1753 return GNUNET_DB_STATUS_HARD_ERROR; 1754 } 1755 process_debits (wa_head); 1756 return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1757 } 1758 1759 1760 /** 1761 * Function called with information about a wire account. Adds the 1762 * account to our list for processing (if it is enabled and we can 1763 * load the plugin). 1764 * 1765 * @param cls closure, NULL 1766 * @param ai account information 1767 */ 1768 static void 1769 process_account_cb (void *cls, 1770 const struct TALER_EXCHANGEDB_AccountInfo *ai) 1771 { 1772 struct WireAccount *wa; 1773 1774 (void) cls; 1775 if ( (! ai->debit_enabled) && 1776 (! ai->credit_enabled) ) 1777 return; /* not an active exchange account */ 1778 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1779 "Found exchange account `%s'\n", 1780 ai->section_name); 1781 wa = GNUNET_new (struct WireAccount); 1782 wa->ai = ai; 1783 GNUNET_CONTAINER_DLL_insert (wa_head, 1784 wa_tail, 1785 wa); 1786 } 1787 1788 1789 /** 1790 * Function called on events received from Postgres. 1791 * 1792 * @param cls closure, NULL 1793 * @param extra additional event data provided 1794 * @param extra_size number of bytes in @a extra 1795 */ 1796 static void 1797 db_notify (void *cls, 1798 const void *extra, 1799 size_t extra_size) 1800 { 1801 (void) cls; 1802 (void) extra; 1803 (void) extra_size; 1804 1805 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1806 "Received notification to wake wire helper\n"); 1807 /* If there are accounts we are still processing, abort 1808 the HTTP requests so we can start afresh. */ 1809 for (struct WireAccount *wa = wa_head; 1810 NULL != wa; 1811 wa = wa->next) 1812 { 1813 if (NULL != wa->dhh) 1814 { 1815 TALER_BANK_debit_history_cancel (wa->dhh); 1816 wa->dhh = NULL; 1817 } 1818 check_exchange_wire_out (wa); 1819 } 1820 1821 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1822 begin_transaction ()) 1823 { 1824 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1825 "Audit failed\n"); 1826 GNUNET_break (0); 1827 global_ret = EXIT_FAILURE; 1828 GNUNET_SCHEDULER_shutdown (); 1829 return; 1830 } 1831 } 1832 1833 1834 /** 1835 * Main function that will be run. 1836 * 1837 * @param cls closure 1838 * @param args remaining command-line arguments 1839 * @param cfgfile name of the configuration file used (for saving, can be NULL!) 1840 * @param c configuration 1841 */ 1842 static void 1843 run (void *cls, 1844 char *const *args, 1845 const char *cfgfile, 1846 const struct GNUNET_CONFIGURATION_Handle *c) 1847 { 1848 (void) cls; 1849 (void) args; 1850 (void) cfgfile; 1851 cfg = c; 1852 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1853 "Launching wire debit auditor\n"); 1854 if (GNUNET_OK != 1855 TALER_ARL_init (c)) 1856 { 1857 global_ret = EXIT_FAILURE; 1858 return; 1859 } 1860 1861 reserve_closures 1862 = GNUNET_CONTAINER_multihashmap_create (1024, 1863 GNUNET_NO); 1864 GNUNET_assert (GNUNET_OK == 1865 TALER_amount_set_zero (TALER_ARL_currency, 1866 &zero)); 1867 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 1868 NULL); 1869 ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, 1870 &rctx); 1871 rctx = GNUNET_CURL_gnunet_rc_create (ctx); 1872 if (NULL == ctx) 1873 { 1874 GNUNET_break (0); 1875 global_ret = EXIT_FAILURE; 1876 return; 1877 } 1878 reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, 1879 GNUNET_NO); 1880 out_map = GNUNET_CONTAINER_multihashmap_create (1024, 1881 true); 1882 if (GNUNET_OK != 1883 TALER_EXCHANGEDB_load_accounts (TALER_ARL_cfg, 1884 TALER_EXCHANGEDB_ALO_DEBIT 1885 | TALER_EXCHANGEDB_ALO_CREDIT 1886 | TALER_EXCHANGEDB_ALO_AUTHDATA)) 1887 { 1888 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1889 "No bank accounts configured\n"); 1890 global_ret = EXIT_NOTCONFIGURED; 1891 GNUNET_SCHEDULER_shutdown (); 1892 return; 1893 } 1894 TALER_EXCHANGEDB_find_accounts (&process_account_cb, 1895 NULL); 1896 1897 if (0 == test_mode) 1898 { 1899 struct GNUNET_DB_EventHeaderP es = { 1900 .size = htons (sizeof (es)), 1901 .type = htons (TALER_DBEVENT_EXCHANGE_AUDITOR_WAKE_HELPER_WIRE) 1902 }; 1903 1904 eh = TALER_AUDITORDB_event_listen (TALER_ARL_adb, 1905 &es, 1906 GNUNET_TIME_UNIT_FOREVER_REL, 1907 &db_notify, 1908 NULL); 1909 GNUNET_assert (NULL != eh); 1910 } 1911 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1912 begin_transaction ()) 1913 { 1914 GNUNET_break (0); 1915 global_ret = EXIT_FAILURE; 1916 GNUNET_SCHEDULER_shutdown (); 1917 return; 1918 } 1919 } 1920 1921 1922 /** 1923 * The main function of the wire auditing tool. Checks that 1924 * the exchange's records of wire transfers match that of 1925 * the wire gateway. 1926 * 1927 * @param argc number of arguments from the command line 1928 * @param argv command line arguments 1929 * @return 0 ok, 1 on error 1930 */ 1931 int 1932 main (int argc, 1933 char *const *argv) 1934 { 1935 const struct GNUNET_GETOPT_CommandLineOption options[] = { 1936 GNUNET_GETOPT_option_flag ('i', 1937 "internal", 1938 "perform checks only applicable for exchange-internal audits", 1939 &internal_checks), 1940 GNUNET_GETOPT_option_flag ('I', 1941 "ignore-not-found", 1942 "continue, even if the bank account of the exchange was not found", 1943 &ignore_account_404), 1944 GNUNET_GETOPT_option_flag ('t', 1945 "test", 1946 "run in test mode and exit when idle", 1947 &test_mode), 1948 GNUNET_GETOPT_option_timetravel ('T', 1949 "timetravel"), 1950 GNUNET_GETOPT_OPTION_END 1951 }; 1952 enum GNUNET_GenericReturnValue ret; 1953 1954 ret = GNUNET_PROGRAM_run ( 1955 TALER_AUDITOR_project_data (), 1956 argc, 1957 argv, 1958 "taler-helper-auditor-wire-debit", 1959 gettext_noop ( 1960 "Audit exchange database for consistency with the bank's outgoing wire transfers"), 1961 options, 1962 &run, 1963 NULL); 1964 if (GNUNET_SYSERR == ret) 1965 return EXIT_INVALIDARGUMENT; 1966 if (GNUNET_NO == ret) 1967 return EXIT_SUCCESS; 1968 return global_ret; 1969 } 1970 1971 1972 /* end of taler-helper-auditor-wire-debit.c */