taler-helper-auditor-wire-debit.c (56396B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2017-2024 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file auditor/taler-helper-auditor-wire-debit.c 18 * @brief audits that wire outgoing transfers match those from an exchange 19 * database. 20 * @author Christian Grothoff 21 * @author Özgür Kesim 22 * 23 * - We check that the outgoing wire transfers match those 24 * given in the 'wire_out' and 'reserve_closures' tables; 25 * any outgoing transfer MUST have a prior justification, 26 * so if one is missing we flag it (and never remove it). 27 * - We check that all wire transfers that should 28 * have been made, were actually made. If any were not made, 29 * we flag those, but may remove those flags if we later 30 * find that the wire transfers were made (wire transfers 31 * could be delayed due to AML/KYC or core-banking issues). 32 */ 33 #include "taler/platform.h" 34 #include <gnunet/gnunet_util_lib.h> 35 #include <gnunet/gnunet_curl_lib.h> 36 #include "taler/taler_auditordb_plugin.h" 37 #include "taler/taler_exchangedb_lib.h" 38 #include "taler/taler_json_lib.h" 39 #include "taler/taler_bank_service.h" 40 #include "taler/taler_signatures.h" 41 #include "report-lib.h" 42 #include "taler/taler_dbevents.h" 43 44 45 /** 46 * Maximum number of wire transfers we process per 47 * (database) transaction. 48 */ 49 #define MAX_PER_TRANSACTION 1024 50 51 /** 52 * How much do we allow the bank and the exchange to disagree about 53 * timestamps? Should be sufficiently large to avoid bogus reports from deltas 54 * created by imperfect clock synchronization and network delay. 55 */ 56 #define TIME_TOLERANCE GNUNET_TIME_relative_multiply ( \ 57 GNUNET_TIME_UNIT_MINUTES, \ 58 15) 59 60 61 /** 62 * How long do we try to long-poll for bank wire transfers? 63 */ 64 #define MAX_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 65 GNUNET_TIME_UNIT_HOURS, \ 66 1) 67 68 69 /** 70 * How long do we wait between polling for bank wire transfers at the minimum? 71 */ 72 #define MIN_LONGPOLL_DELAY GNUNET_TIME_relative_multiply ( \ 73 GNUNET_TIME_UNIT_MINUTES, \ 74 5) 75 76 77 /** 78 * Run in test mode. Exit when idle instead of 79 * going to sleep and waiting for more work. 80 */ 81 static int test_mode; 82 83 84 /** 85 * Information we keep for each supported account. 86 */ 87 struct WireAccount 88 { 89 /** 90 * Accounts are kept in a DLL. 91 */ 92 struct WireAccount *next; 93 94 /** 95 * Plugins are kept in a DLL. 96 */ 97 struct WireAccount *prev; 98 99 /** 100 * Account details. 101 */ 102 const struct TALER_EXCHANGEDB_AccountInfo *ai; 103 104 /** 105 * Active wire request for the transaction history. 106 */ 107 struct TALER_BANK_DebitHistoryHandle *dhh; 108 109 /** 110 * Task to trigger @e dhh long-polling. 111 */ 112 struct GNUNET_SCHEDULER_Task *dhh_task; 113 114 /** 115 * Time when we expect the current @e dhh long-poll 116 * to finish and we thus could begin another one. 117 */ 118 struct GNUNET_TIME_Absolute dhh_next; 119 120 /** 121 * Progress point for this account. 122 */ 123 uint64_t last_wire_out_serial_id; 124 125 /** 126 * Initial progress point for this account. 127 */ 128 uint64_t start_wire_out_serial_id; 129 130 /** 131 * Where we are in the outbound transaction history. 132 */ 133 uint64_t wire_off_out; 134 135 /** 136 * Label under which we store our pp's reserve_in_serial_id. 137 */ 138 char *label_wire_out_serial_id; 139 140 /** 141 * Label under which we store our wire_off_out. 142 */ 143 char *label_wire_off_out; 144 }; 145 146 147 /** 148 * Information we track for a reserve being closed. 149 */ 150 struct ReserveClosure 151 { 152 /** 153 * Row in the reserves_closed table for this action. 154 */ 155 uint64_t rowid; 156 157 /** 158 * When was the reserve closed? 159 */ 160 struct GNUNET_TIME_Timestamp execution_date; 161 162 /** 163 * Amount transferred (amount remaining minus fee). 164 */ 165 struct TALER_Amount amount; 166 167 /** 168 * Target account where the money was sent. 169 */ 170 struct TALER_FullPayto receiver_account; 171 172 /** 173 * Wire transfer subject used. 174 */ 175 struct TALER_WireTransferIdentifierRawP wtid; 176 }; 177 178 179 /** 180 * Map from H(wtid,receiver_account) to `struct ReserveClosure` entries. 181 */ 182 static struct GNUNET_CONTAINER_MultiHashMap *reserve_closures; 183 184 /** 185 * Return value from main(). 186 */ 187 static int global_ret; 188 189 /** 190 * State of the current database transaction with 191 * the auditor DB. 192 */ 193 static enum GNUNET_DB_QueryStatus global_qs; 194 195 /** 196 * Map with information about outgoing wire transfers. 197 * Maps hashes of the wire subjects (in binary encoding) 198 * to `struct ReserveOutInfo`s. 199 */ 200 static struct GNUNET_CONTAINER_MultiHashMap *out_map; 201 202 /** 203 * Head of list of wire accounts we still need to look at. 204 */ 205 static struct WireAccount *wa_head; 206 207 /** 208 * Tail of list of wire accounts we still need to look at. 209 */ 210 static struct WireAccount *wa_tail; 211 212 /** 213 * Last reserve_out / wire_out serial IDs seen. 214 */ 215 static TALER_ARL_DEF_PP (wire_reserve_close_id); 216 217 /** 218 * Total amount that was transferred too much from the exchange. 219 */ 220 static TALER_ARL_DEF_AB (total_bad_amount_out_plus); 221 222 /** 223 * Total amount that was transferred too little from the exchange. 224 */ 225 static TALER_ARL_DEF_AB (total_bad_amount_out_minus); 226 227 /** 228 * Total amount of reserve closures which the exchange did not transfer in time. 229 */ 230 static TALER_ARL_DEF_AB (total_closure_amount_lag); 231 232 /** 233 * Total amount affected by duplicate wire transfer 234 * subjects. 235 */ 236 static TALER_ARL_DEF_AB (wire_debit_duplicate_transfer_subject_total); 237 238 /** 239 * Total amount debited to exchange accounts. 240 */ 241 static TALER_ARL_DEF_AB (total_wire_out); 242 243 /** 244 * Total amount of profits drained. 245 */ 246 static TALER_ARL_DEF_AB (total_drained); 247 248 /** 249 * Amount of zero in our currency. 250 */ 251 static struct TALER_Amount zero; 252 253 /** 254 * Handle to the context for interacting with the bank. 255 */ 256 static struct GNUNET_CURL_Context *ctx; 257 258 /** 259 * Scheduler context for running the @e ctx. 260 */ 261 static struct GNUNET_CURL_RescheduleContext *rctx; 262 263 /** 264 * Should we run checks that only work for exchange-internal audits? 265 */ 266 static int internal_checks; 267 268 /** 269 * Should we ignore if the bank does not know our bank 270 * account? 271 */ 272 static int ignore_account_404; 273 274 /** 275 * Database event handler to wake us up again. 276 */ 277 static struct GNUNET_DB_EventHandler *eh; 278 279 /** 280 * The auditors's configuration. 281 */ 282 static const struct GNUNET_CONFIGURATION_Handle *cfg; 283 284 285 /** 286 * Entry in map with wire information we expect to obtain from the 287 * #TALER_ARL_edb later. 288 */ 289 struct WireTransferOutInfo 290 { 291 292 /** 293 * Hash of the wire transfer subject. 294 */ 295 struct GNUNET_HashCode subject_hash; 296 297 /** 298 * Expected details about the wire transfer. 299 */ 300 struct TALER_BANK_DebitDetails details; 301 302 }; 303 304 305 /** 306 * Free entry in #out_map. 307 * 308 * @param cls NULL 309 * @param key unused key 310 * @param value the `struct WireTransferOutInfo` to free 311 * @return #GNUNET_OK 312 */ 313 static enum GNUNET_GenericReturnValue 314 free_roi (void *cls, 315 const struct GNUNET_HashCode *key, 316 void *value) 317 { 318 struct WireTransferOutInfo *roi = value; 319 320 (void) cls; 321 GNUNET_assert (GNUNET_YES == 322 GNUNET_CONTAINER_multihashmap_remove (out_map, 323 key, 324 roi)); 325 GNUNET_free (roi); 326 return GNUNET_OK; 327 } 328 329 330 /** 331 * Free entry in #reserve_closures. 332 * 333 * @param cls NULL 334 * @param key unused key 335 * @param value the `struct ReserveClosure` to free 336 * @return #GNUNET_OK 337 */ 338 static enum GNUNET_GenericReturnValue 339 free_rc (void *cls, 340 const struct GNUNET_HashCode *key, 341 void *value) 342 { 343 struct ReserveClosure *rc = value; 344 345 (void) cls; 346 GNUNET_assert (GNUNET_YES == 347 GNUNET_CONTAINER_multihashmap_remove (reserve_closures, 348 key, 349 rc)); 350 GNUNET_free (rc->receiver_account.full_payto); 351 GNUNET_free (rc); 352 return GNUNET_OK; 353 } 354 355 356 /** 357 * Task run on shutdown. 358 * 359 * @param cls NULL 360 */ 361 static void 362 do_shutdown (void *cls) 363 { 364 struct WireAccount *wa; 365 366 (void) cls; 367 if (NULL != eh) 368 { 369 TALER_ARL_adb->event_listen_cancel (eh); 370 eh = NULL; 371 } 372 TALER_ARL_done (); 373 if (NULL != reserve_closures) 374 { 375 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 376 &free_rc, 377 NULL); 378 GNUNET_CONTAINER_multihashmap_destroy (reserve_closures); 379 reserve_closures = NULL; 380 } 381 if (NULL != out_map) 382 { 383 GNUNET_CONTAINER_multihashmap_iterate (out_map, 384 &free_roi, 385 NULL); 386 GNUNET_CONTAINER_multihashmap_destroy (out_map); 387 out_map = NULL; 388 } 389 while (NULL != (wa = wa_head)) 390 { 391 if (NULL != wa->dhh_task) 392 { 393 GNUNET_SCHEDULER_cancel (wa->dhh_task); 394 wa->dhh_task = NULL; 395 } 396 if (NULL != wa->dhh) 397 { 398 TALER_BANK_debit_history_cancel (wa->dhh); 399 wa->dhh = NULL; 400 } 401 GNUNET_CONTAINER_DLL_remove (wa_head, 402 wa_tail, 403 wa); 404 GNUNET_free (wa->label_wire_out_serial_id); 405 GNUNET_free (wa->label_wire_off_out); 406 GNUNET_free (wa); 407 } 408 if (NULL != ctx) 409 { 410 GNUNET_CURL_fini (ctx); 411 ctx = NULL; 412 } 413 if (NULL != rctx) 414 { 415 GNUNET_CURL_gnunet_rc_destroy (rctx); 416 rctx = NULL; 417 } 418 TALER_EXCHANGEDB_unload_accounts (); 419 TALER_ARL_cfg = NULL; 420 } 421 422 423 /** 424 * Detect any entries in #reserve_closures that were not yet 425 * observed on the wire transfer side and update the progress 426 * point accordingly. 427 * 428 * @param cls NULL 429 * @param key unused key 430 * @param value the `struct ReserveClosure` to free 431 * @return #GNUNET_OK 432 */ 433 static enum GNUNET_GenericReturnValue 434 check_pending_rc (void *cls, 435 const struct GNUNET_HashCode *key, 436 void *value) 437 { 438 struct ReserveClosure *rc = value; 439 440 (void) cls; 441 (void) key; 442 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 443 "Missing wire transfer for closed reserve with balance %s\n", 444 TALER_amount2s (&rc->amount)); 445 if (! TALER_amount_is_zero (&rc->amount)) 446 { 447 struct TALER_AUDITORDB_ClosureLags cl = { 448 .problem_row_id = rc->rowid, 449 .account = rc->receiver_account, 450 .amount = rc->amount, 451 .deadline = rc->execution_date.abs_time, 452 .wtid = rc->wtid 453 }; 454 enum GNUNET_DB_QueryStatus qs; 455 456 qs = TALER_ARL_adb->insert_auditor_closure_lags ( 457 TALER_ARL_adb->cls, 458 &cl); 459 if (qs < 0) 460 { 461 global_qs = qs; 462 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 463 return GNUNET_SYSERR; 464 } 465 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_closure_amount_lag), 466 &TALER_ARL_USE_AB (total_closure_amount_lag), 467 &rc->amount); 468 } 469 return GNUNET_OK; 470 } 471 472 473 /** 474 * Compute the key under which a reserve closure for a given 475 * @a receiver_account and @a wtid would be stored. 476 * 477 * @param receiver_account payto://-URI of the account 478 * @param wtid wire transfer identifier used 479 * @param[out] key set to the key 480 */ 481 static void 482 hash_rc (const struct TALER_FullPayto receiver_account, 483 const struct TALER_WireTransferIdentifierRawP *wtid, 484 struct GNUNET_HashCode *key) 485 { 486 struct TALER_NormalizedPayto npto 487 = TALER_payto_normalize (receiver_account); 488 size_t slen = strlen (npto.normalized_payto); 489 char buf[sizeof (struct TALER_WireTransferIdentifierRawP) + slen]; 490 491 GNUNET_memcpy (buf, 492 wtid, 493 sizeof (*wtid)); 494 GNUNET_memcpy (&buf[sizeof (*wtid)], 495 npto.normalized_payto, 496 slen); 497 GNUNET_CRYPTO_hash (buf, 498 sizeof (buf), 499 key); 500 GNUNET_free (npto.normalized_payto); 501 } 502 503 504 /** 505 * Start the database transactions and begin the audit. 506 * 507 * @return transaction status code 508 */ 509 static enum GNUNET_DB_QueryStatus 510 begin_transaction (void); 511 512 513 /** 514 * Commit the transaction, checkpointing our progress in the auditor DB. 515 * 516 * @param qs transaction status so far 517 */ 518 static void 519 commit (enum GNUNET_DB_QueryStatus qs) 520 { 521 GNUNET_CONTAINER_multihashmap_iterate (reserve_closures, 522 &check_pending_rc, 523 NULL); 524 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 525 "Transaction logic ended with status %d\n", 526 qs); 527 TALER_ARL_edb->rollback (TALER_ARL_edb->cls); 528 qs = TALER_ARL_adb->update_balance ( 529 TALER_ARL_adb->cls, 530 TALER_ARL_SET_AB (total_drained), 531 TALER_ARL_SET_AB (total_wire_out), 532 TALER_ARL_SET_AB (total_bad_amount_out_plus), 533 TALER_ARL_SET_AB (total_bad_amount_out_minus), 534 TALER_ARL_SET_AB (total_closure_amount_lag), 535 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 536 TALER_ARL_SET_AB (total_wire_out), 537 NULL); 538 if (0 > qs) 539 goto handle_db_error; 540 qs = TALER_ARL_adb->insert_balance ( 541 TALER_ARL_adb->cls, 542 TALER_ARL_SET_AB (total_drained), 543 TALER_ARL_SET_AB (total_wire_out), 544 TALER_ARL_SET_AB (total_bad_amount_out_plus), 545 TALER_ARL_SET_AB (total_bad_amount_out_minus), 546 TALER_ARL_SET_AB (total_closure_amount_lag), 547 TALER_ARL_SET_AB (wire_debit_duplicate_transfer_subject_total), 548 TALER_ARL_SET_AB (total_wire_out), 549 NULL); 550 if (0 > qs) 551 goto handle_db_error; 552 for (struct WireAccount *wa = wa_head; 553 NULL != wa; 554 wa = wa->next) 555 { 556 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 557 "Transaction of account %s ends at %llu/%llu\n", 558 wa->ai->section_name, 559 (unsigned long long) wa->last_wire_out_serial_id, 560 (unsigned long long) wa->wire_off_out); 561 qs = TALER_ARL_adb->update_auditor_progress ( 562 TALER_ARL_adb->cls, 563 wa->label_wire_out_serial_id, 564 wa->last_wire_out_serial_id, 565 wa->label_wire_off_out, 566 wa->wire_off_out, 567 NULL); 568 if (0 > qs) 569 goto handle_db_error; 570 qs = TALER_ARL_adb->insert_auditor_progress ( 571 TALER_ARL_adb->cls, 572 wa->label_wire_out_serial_id, 573 wa->last_wire_out_serial_id, 574 wa->label_wire_off_out, 575 wa->wire_off_out, 576 NULL); 577 if (0 > qs) 578 goto handle_db_error; 579 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 580 "Transaction ends at %s=%llu for account `%s'\n", 581 wa->label_wire_out_serial_id, 582 (unsigned long long) wa->last_wire_out_serial_id, 583 wa->ai->section_name); 584 } 585 qs = TALER_ARL_adb->update_auditor_progress ( 586 TALER_ARL_adb->cls, 587 TALER_ARL_SET_PP (wire_reserve_close_id), 588 NULL); 589 if (0 > qs) 590 goto handle_db_error; 591 qs = TALER_ARL_adb->insert_auditor_progress ( 592 TALER_ARL_adb->cls, 593 TALER_ARL_SET_PP (wire_reserve_close_id), 594 NULL); 595 if (0 > qs) 596 goto handle_db_error; 597 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 598 "Concluded audit step at %llu\n", 599 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 600 qs = TALER_ARL_adb->commit (TALER_ARL_adb->cls); 601 if (0 > qs) 602 goto handle_db_error; 603 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 604 "Transaction concluded!\n"); 605 if (1 == test_mode) 606 GNUNET_SCHEDULER_shutdown (); 607 return; 608 handle_db_error: 609 TALER_ARL_adb->rollback (TALER_ARL_adb->cls); 610 for (unsigned int max_retries = 3; max_retries>0; max_retries--) 611 { 612 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 613 break; 614 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 615 "Serialization issue, trying again\n"); 616 qs = begin_transaction (); 617 } 618 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 619 "Hard database error, terminating\n"); 620 GNUNET_SCHEDULER_shutdown (); 621 } 622 623 624 /** 625 * Check that @a want is within #TIME_TOLERANCE of @a have. 626 * Otherwise report an inconsistency in row @a rowid of @a table. 627 * 628 * @param table where is the inconsistency (if any) 629 * @param rowid what is the row 630 * @param want what is the expected time 631 * @param have what is the time we got 632 * @return true on success, false to abort 633 */ 634 static bool 635 check_time_difference (const char *table, 636 uint64_t rowid, 637 struct GNUNET_TIME_Timestamp want, 638 struct GNUNET_TIME_Timestamp have) 639 { 640 struct GNUNET_TIME_Relative delta; 641 char *details; 642 643 if (GNUNET_TIME_timestamp_cmp (have, >, want)) 644 delta = GNUNET_TIME_absolute_get_difference (want.abs_time, 645 have.abs_time); 646 else 647 delta = GNUNET_TIME_absolute_get_difference (have.abs_time, 648 want.abs_time); 649 if (GNUNET_TIME_relative_cmp (delta, 650 <=, 651 TIME_TOLERANCE)) 652 return true; 653 654 GNUNET_asprintf (&details, 655 "execution date mismatch (%s)", 656 GNUNET_TIME_relative2s (delta, 657 true)); 658 { 659 struct TALER_AUDITORDB_RowMinorInconsistencies rmi = { 660 .row_table = (char *) table, 661 .problem_row = rowid, 662 .diagnostic = details 663 }; 664 enum GNUNET_DB_QueryStatus qs; 665 666 qs = TALER_ARL_adb->insert_row_minor_inconsistencies ( 667 TALER_ARL_adb->cls, 668 &rmi); 669 670 if (qs < 0) 671 { 672 global_qs = qs; 673 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 674 GNUNET_free (details); 675 return false; 676 } 677 } 678 GNUNET_free (details); 679 return true; 680 } 681 682 683 /** 684 * Closure for #check_rc_matches 685 */ 686 struct CheckMatchContext 687 { 688 689 /** 690 * Reserve operation looking for a match 691 */ 692 const struct WireTransferOutInfo *roi; 693 694 /** 695 * Set to true if we found a match. 696 */ 697 bool found; 698 }; 699 700 701 /** 702 * Check if any of the reserve closures match the given wire transfer. 703 * 704 * @param[in,out] cls a `struct CheckMatchContext` 705 * @param key key of @a value in #reserve_closures 706 * @param value a `struct ReserveClosure` 707 */ 708 static enum GNUNET_GenericReturnValue 709 check_rc_matches (void *cls, 710 const struct GNUNET_HashCode *key, 711 void *value) 712 { 713 struct CheckMatchContext *cmx = cls; 714 struct ReserveClosure *rc = value; 715 716 if ((0 == GNUNET_memcmp (&cmx->roi->details.wtid, 717 &rc->wtid)) && 718 (0 == TALER_full_payto_cmp (rc->receiver_account, 719 cmx->roi->details.credit_account_uri)) && 720 (0 == TALER_amount_cmp (&rc->amount, 721 &cmx->roi->details.amount))) 722 { 723 if (! check_time_difference ("reserves_closures", 724 rc->rowid, 725 rc->execution_date, 726 cmx->roi->details.execution_date)) 727 { 728 free_rc (NULL, 729 key, 730 rc); 731 return GNUNET_SYSERR; 732 } 733 cmx->found = true; 734 free_rc (NULL, 735 key, 736 rc); 737 return GNUNET_NO; 738 } 739 return GNUNET_OK; 740 } 741 742 743 /** 744 * Maximum required length for make_missing_diag(). 745 */ 746 #define MAX_DIAG_LEN 128 747 748 /** 749 * Make diagnostic string for missing wire transfer. 750 * 751 * @param[out] diag where to write the diagnostic string 752 * @param wtid wire transfer ID to include 753 */ 754 static void 755 make_missing_diag (char diag[MAX_DIAG_LEN], 756 const struct TALER_WireTransferIdentifierRawP *wtid) 757 { 758 char *wtid_s; 759 760 wtid_s = GNUNET_STRINGS_data_to_string_alloc (wtid, 761 sizeof (*wtid)); 762 GNUNET_snprintf (diag, 763 MAX_DIAG_LEN, 764 "expected outgoing wire transfer %s missing", 765 wtid_s); 766 GNUNET_free (wtid_s); 767 } 768 769 770 /** 771 * Check if an existing report on a missing wire 772 * out operation justified the @a roi. If so, 773 * clear the existing report. 774 * 775 * @param roi reserve out operation to check 776 * @return #GNUNET_YES if @a roi was justified by a previous report, 777 * #GNUNET_NO of @a roi was not justified by a previous missing report 778 * #GNUNET_SYSERR on database trouble 779 */ 780 static enum GNUNET_GenericReturnValue 781 check_reported_inconsistency (struct WireTransferOutInfo *roi) 782 { 783 char diag[MAX_DIAG_LEN]; 784 struct TALER_AUDITORDB_WireOutInconsistency woi = { 785 .wire_out_row_id = roi->details.serial_id, 786 .destination_account = roi->details.credit_account_uri, 787 .diagnostic = diag, 788 .expected = roi->details.amount, 789 .claimed = zero, 790 }; 791 enum GNUNET_DB_QueryStatus qs; 792 793 make_missing_diag (diag, 794 &roi->details.wtid); 795 qs = TALER_ARL_adb->delete_wire_out_inconsistency_if_matching ( 796 TALER_ARL_adb->cls, 797 &woi); 798 if (qs < 0) 799 { 800 global_qs = qs; 801 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 802 return GNUNET_SYSERR; 803 } 804 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 805 { 806 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 807 "Deletion of wire out inconsistency %llu (%s, %s, %s) failed: not reported missing!\n", 808 (unsigned long long) roi->details.serial_id, 809 roi->details.credit_account_uri.full_payto, 810 diag, 811 TALER_amount2s (&roi->details.amount)); 812 return GNUNET_NO; 813 } 814 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 815 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 816 &roi->details.amount); 817 return GNUNET_YES; 818 } 819 820 821 /** 822 * Check if a profit drain operation justified the @a roi 823 * 824 * @param roi reserve out operation to check 825 * @return #GNUNET_YES if @a roi was justified by a profit drain, 826 * #GNUNET_NO of @a roi was not justified by a proft drain 827 * #GNUNET_SYSERR on database trouble 828 */ 829 static enum GNUNET_GenericReturnValue 830 check_profit_drain (struct WireTransferOutInfo *roi) 831 { 832 enum GNUNET_DB_QueryStatus qs; 833 uint64_t serial; 834 char *account_section; 835 struct TALER_FullPayto payto_uri; 836 struct GNUNET_TIME_Timestamp request_timestamp; 837 struct TALER_Amount amount; 838 struct TALER_MasterSignatureP master_sig; 839 840 qs = TALER_ARL_edb->get_drain_profit ( 841 TALER_ARL_edb->cls, 842 &roi->details.wtid, 843 &serial, 844 &account_section, 845 &payto_uri, 846 &request_timestamp, 847 &amount, 848 &master_sig); 849 switch (qs) 850 { 851 case GNUNET_DB_STATUS_HARD_ERROR: 852 GNUNET_break (0); 853 global_ret = EXIT_FAILURE; 854 GNUNET_SCHEDULER_shutdown (); 855 return GNUNET_SYSERR; 856 case GNUNET_DB_STATUS_SOFT_ERROR: 857 /* should fail on commit later ... */ 858 GNUNET_break (0); 859 return GNUNET_SYSERR; 860 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 861 /* not a profit drain */ 862 return GNUNET_NO; 863 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 864 break; 865 } 866 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 867 "Profit drain of %s to %s found!\n", 868 TALER_amount2s (&amount), 869 payto_uri.full_payto); 870 if (GNUNET_OK != 871 TALER_exchange_offline_profit_drain_verify ( 872 &roi->details.wtid, 873 request_timestamp, 874 &amount, 875 account_section, 876 payto_uri, 877 &TALER_ARL_master_pub, 878 &master_sig)) 879 { 880 struct TALER_AUDITORDB_RowInconsistency ri = { 881 .row_id = roi->details.serial_id, 882 .row_table = (char *) "profit_drains", 883 .diagnostic = (char *) "invalid signature" 884 }; 885 886 GNUNET_break (0); 887 qs = TALER_ARL_adb->insert_row_inconsistency ( 888 TALER_ARL_adb->cls, 889 &ri); 890 GNUNET_free (payto_uri.full_payto); 891 GNUNET_free (account_section); 892 if (qs < 0) 893 { 894 global_qs = qs; 895 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 896 return GNUNET_SYSERR; 897 } 898 return GNUNET_NO; 899 } 900 GNUNET_free (account_section); 901 902 { 903 if (0 != 904 TALER_full_payto_normalize_and_cmp (payto_uri, 905 roi->details.credit_account_uri)) 906 { 907 struct TALER_AUDITORDB_WireOutInconsistency woi = { 908 .wire_out_row_id = serial, 909 .destination_account = roi->details.credit_account_uri, 910 .diagnostic = (char *) "profit drain wired to invalid account", 911 .expected = roi->details.amount, 912 .claimed = zero, 913 }; 914 915 qs = TALER_ARL_adb->insert_wire_out_inconsistency ( 916 TALER_ARL_adb->cls, 917 &woi); 918 if (qs < 0) 919 { 920 global_qs = qs; 921 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 922 GNUNET_free (payto_uri.full_payto); 923 return GNUNET_SYSERR; 924 } 925 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 926 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 927 &amount); 928 GNUNET_free (payto_uri.full_payto); 929 return GNUNET_YES; /* justified, kind-of */ 930 } 931 } 932 GNUNET_free (payto_uri.full_payto); 933 if (0 != 934 TALER_amount_cmp (&amount, 935 &roi->details.amount)) 936 { 937 struct TALER_AUDITORDB_WireOutInconsistency woi = { 938 .wire_out_row_id = roi->details.serial_id, 939 .destination_account = roi->details.credit_account_uri, 940 .diagnostic = (char *) "incorrect amount drained to correct account", 941 .expected = roi->details.amount, 942 .claimed = amount, 943 }; 944 945 qs = TALER_ARL_adb->insert_wire_out_inconsistency ( 946 TALER_ARL_adb->cls, 947 &woi); 948 if (qs < 0) 949 { 950 global_qs = qs; 951 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 952 return GNUNET_SYSERR; 953 } 954 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 955 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 956 &roi->details.amount); 957 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 958 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 959 &amount); 960 return GNUNET_YES; /* justified, kind-of */ 961 } 962 /* profit drain was correct */ 963 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_drained), 964 &TALER_ARL_USE_AB (total_drained), 965 &amount); 966 return GNUNET_YES; 967 } 968 969 970 /** 971 * Check whether the given transfer was justified because it was 972 * actually previously reported as missing. 973 * 974 * @param roi the reserve out operation to check 975 * @return #GNUNET_OK on success, #GNUNET_NO if there was no 976 * matching lag, #GNUNET_SYSERR on database trouble 977 */ 978 static enum GNUNET_GenericReturnValue 979 check_closure_lag (const struct WireTransferOutInfo *roi) 980 { 981 enum GNUNET_DB_QueryStatus qs; 982 983 qs = TALER_ARL_adb->delete_auditor_closure_lag ( 984 TALER_ARL_adb->cls, 985 &roi->details.amount, 986 &roi->details.wtid, 987 roi->details.credit_account_uri); 988 if (qs < 0) 989 { 990 global_qs = qs; 991 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 992 return GNUNET_SYSERR; 993 } 994 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 995 return GNUNET_NO; 996 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 997 "Cleared closure lag: found justification\n"); 998 TALER_ARL_amount_subtract (&TALER_ARL_USE_AB (total_closure_amount_lag), 999 &TALER_ARL_USE_AB (total_closure_amount_lag), 1000 &roi->details.amount); 1001 return GNUNET_YES; /* found! */ 1002 } 1003 1004 1005 /** 1006 * Check whether the given transfer was justified by a reserve closure or 1007 * profit drain. If not, complain that we failed to match an entry from 1008 * #out_map. This means a wire transfer was made without proper 1009 * justification. 1010 * 1011 * @param cls a `struct WireAccount` 1012 * @param key unused key 1013 * @param value the `struct WireTransferOutInfo` to report 1014 * @return #GNUNET_OK on success 1015 */ 1016 static enum GNUNET_GenericReturnValue 1017 complain_out_not_found (void *cls, 1018 const struct GNUNET_HashCode *key, 1019 void *value) 1020 { 1021 // struct WireAccount *wa = cls; 1022 struct WireTransferOutInfo *roi = value; 1023 struct GNUNET_HashCode rkey; 1024 struct CheckMatchContext cmx = { 1025 .roi = roi, 1026 .found = false 1027 }; 1028 enum GNUNET_GenericReturnValue ret; 1029 1030 (void) cls; 1031 (void) key; 1032 hash_rc (roi->details.credit_account_uri, 1033 &roi->details.wtid, 1034 &rkey); 1035 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1036 "Checking for reserve closure %s benefiting %s\n", 1037 GNUNET_h2s (&rkey), 1038 roi->details.credit_account_uri.full_payto); 1039 GNUNET_CONTAINER_multihashmap_get_multiple (reserve_closures, 1040 &rkey, 1041 &check_rc_matches, 1042 &cmx); 1043 if (cmx.found) 1044 return GNUNET_OK; 1045 ret = check_reported_inconsistency (roi); 1046 if (GNUNET_NO != ret) 1047 return ret; 1048 ret = check_profit_drain (roi); 1049 if (GNUNET_NO != ret) 1050 return ret; 1051 ret = check_closure_lag (roi); 1052 if (GNUNET_NO != ret) 1053 return ret; 1054 1055 { 1056 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1057 .destination_account = roi->details.credit_account_uri, 1058 .diagnostic = (char *) "missing justification for outgoing wire transfer", 1059 .wire_out_row_id = roi->details.serial_id, 1060 .expected = zero, 1061 .claimed = roi->details.amount 1062 }; 1063 enum GNUNET_DB_QueryStatus qs; 1064 1065 qs = TALER_ARL_adb->insert_wire_out_inconsistency ( 1066 TALER_ARL_adb->cls, 1067 &woi); 1068 if (qs < 0) 1069 { 1070 global_qs = qs; 1071 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1072 return GNUNET_SYSERR; 1073 } 1074 } 1075 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1076 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1077 &roi->details.amount); 1078 return GNUNET_OK; 1079 } 1080 1081 1082 /** 1083 * Function called with details about outgoing wire transfers 1084 * as claimed by the exchange DB. 1085 * 1086 * @param cls a `struct WireAccount` 1087 * @param rowid unique serial ID in wire_out table 1088 * @param date timestamp of the transfer (roughly) 1089 * @param wtid wire transfer subject 1090 * @param payto_uri wire transfer details of the receiver 1091 * @param amount amount that was wired 1092 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1093 */ 1094 static enum GNUNET_GenericReturnValue 1095 wire_out_cb ( 1096 void *cls, 1097 uint64_t rowid, 1098 struct GNUNET_TIME_Timestamp date, 1099 const struct TALER_WireTransferIdentifierRawP *wtid, 1100 const struct TALER_FullPayto payto_uri, 1101 const struct TALER_Amount *amount) 1102 { 1103 struct WireAccount *wa = cls; 1104 struct GNUNET_HashCode key; 1105 struct WireTransferOutInfo *roi; 1106 1107 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1108 "Exchange wire OUT #%llu at %s of %s with WTID %s\n", 1109 (unsigned long long) rowid, 1110 GNUNET_TIME_timestamp2s (date), 1111 TALER_amount2s (amount), 1112 TALER_B2S (wtid)); 1113 wa->last_wire_out_serial_id = rowid + 1; 1114 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_wire_out), 1115 &TALER_ARL_USE_AB (total_wire_out), 1116 amount); 1117 GNUNET_CRYPTO_hash (wtid, 1118 sizeof (*wtid), 1119 &key); 1120 roi = GNUNET_CONTAINER_multihashmap_get (out_map, 1121 &key); 1122 if (NULL == roi) 1123 { 1124 /* Wire transfer was not made (yet) at all (but would have been 1125 justified), so the entire amount is missing / still to be done. This 1126 is moderately harmless, it might just be that the 1127 taler-exchange-transfer tool or bank has not yet fully caught up with 1128 the transfers it should do. 1129 May be cleared later by check_reported_inconsistency() */ 1130 char diag[MAX_DIAG_LEN]; 1131 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1132 .destination_account = payto_uri, 1133 .diagnostic = diag, 1134 .wire_out_row_id = rowid, 1135 .expected = *amount, 1136 .claimed = zero, 1137 }; 1138 enum GNUNET_DB_QueryStatus qs; 1139 1140 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1141 "Wire out for row %llu still missing\n", 1142 (unsigned long long) rowid); 1143 make_missing_diag (diag, 1144 wtid); 1145 qs = TALER_ARL_adb->insert_wire_out_inconsistency ( 1146 TALER_ARL_adb->cls, 1147 &woi); 1148 if (qs < 0) 1149 { 1150 global_qs = qs; 1151 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1152 return GNUNET_SYSERR; 1153 } 1154 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1155 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1156 amount); 1157 return GNUNET_OK; 1158 } 1159 1160 if (0 != TALER_full_payto_normalize_and_cmp (payto_uri, 1161 roi->details.credit_account_uri)) 1162 { 1163 /* Destination bank account is wrong in actual wire transfer, so 1164 we should count the wire transfer as entirely spurious, and 1165 additionally consider the justified wire transfer as missing. */ 1166 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1167 .wire_out_row_id = rowid, 1168 .destination_account = payto_uri, 1169 .diagnostic = (char *) "receiver account mismatch", 1170 .expected = *amount, 1171 .claimed = roi->details.amount, 1172 }; 1173 enum GNUNET_DB_QueryStatus qs; 1174 1175 qs = TALER_ARL_adb->insert_wire_out_inconsistency ( 1176 TALER_ARL_adb->cls, 1177 &woi); 1178 if (qs < 0) 1179 { 1180 global_qs = qs; 1181 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1182 return GNUNET_SYSERR; 1183 } 1184 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1185 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1186 &roi->details.amount); 1187 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1188 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1189 amount); 1190 GNUNET_assert (GNUNET_OK == 1191 free_roi (NULL, 1192 &key, 1193 roi)); 1194 return GNUNET_OK; 1195 } 1196 1197 if (0 != TALER_amount_cmp (&roi->details.amount, 1198 amount)) 1199 { 1200 struct TALER_AUDITORDB_WireOutInconsistency woi = { 1201 .destination_account = payto_uri, 1202 .diagnostic = (char *) "wire amount does not match", 1203 .wire_out_row_id = rowid, 1204 .expected = *amount, 1205 .claimed = roi->details.amount, 1206 }; 1207 enum GNUNET_DB_QueryStatus qs; 1208 1209 qs = TALER_ARL_adb->insert_wire_out_inconsistency ( 1210 TALER_ARL_adb->cls, 1211 &woi); 1212 if (qs < 0) 1213 { 1214 global_qs = qs; 1215 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1216 return GNUNET_SYSERR; 1217 } 1218 if (0 < TALER_amount_cmp (amount, 1219 &roi->details.amount)) 1220 { 1221 /* amount > roi->details.amount: wire transfer was smaller than it should have been */ 1222 struct TALER_Amount delta; 1223 1224 TALER_ARL_amount_subtract (&delta, 1225 amount, 1226 &roi->details.amount); 1227 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_minus), 1228 &TALER_ARL_USE_AB (total_bad_amount_out_minus), 1229 &delta); 1230 } 1231 else 1232 { 1233 /* roi->details.amount < amount: wire transfer was larger than it should have been */ 1234 struct TALER_Amount delta; 1235 1236 TALER_ARL_amount_subtract (&delta, 1237 &roi->details.amount, 1238 amount); 1239 TALER_ARL_amount_add (&TALER_ARL_USE_AB (total_bad_amount_out_plus), 1240 &TALER_ARL_USE_AB (total_bad_amount_out_plus), 1241 &delta); 1242 } 1243 GNUNET_assert (GNUNET_OK == 1244 free_roi (NULL, 1245 &key, 1246 roi)); 1247 return GNUNET_OK; 1248 } 1249 1250 { 1251 enum GNUNET_GenericReturnValue ret; 1252 1253 if (! check_time_difference ("wire_out", 1254 rowid, 1255 date, 1256 roi->details.execution_date)) 1257 { 1258 /* We had a database error, fail */ 1259 ret = GNUNET_SYSERR; 1260 } 1261 else 1262 { 1263 ret = GNUNET_OK; 1264 } 1265 GNUNET_assert (GNUNET_OK == 1266 free_roi (NULL, 1267 &key, 1268 roi)); 1269 return ret; 1270 } 1271 } 1272 1273 1274 /** 1275 * Main function for processing 'debit' data. We start by going over 1276 * the DEBIT transactions this time, and then verify that all of them are 1277 * justified by reserve closures, profit drains or regular outgoing 1278 * wire transfers from aggregated deposits. 1279 * 1280 * @param[in,out] wa wire account list to process 1281 */ 1282 static void 1283 process_debits (struct WireAccount *wa); 1284 1285 1286 /** 1287 * Go over the "wire_out" table of the exchange and 1288 * verify that all wire outs are in that table. 1289 * 1290 * @param[in,out] wa wire account we are processing 1291 */ 1292 static void 1293 check_exchange_wire_out (struct WireAccount *wa) 1294 { 1295 enum GNUNET_DB_QueryStatus qs; 1296 1297 GNUNET_assert (NULL == wa->dhh); 1298 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1299 "Analyzing exchange's wire OUT table for account `%s'\n", 1300 wa->ai->section_name); 1301 qs = TALER_ARL_edb->select_wire_out_above_serial_id_by_account ( 1302 TALER_ARL_edb->cls, 1303 wa->ai->section_name, 1304 wa->last_wire_out_serial_id, 1305 &wire_out_cb, 1306 wa); 1307 if (0 > qs) 1308 { 1309 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1310 global_ret = EXIT_FAILURE; 1311 GNUNET_SCHEDULER_shutdown (); 1312 return; 1313 } 1314 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1315 &complain_out_not_found, 1316 wa); 1317 /* clean up */ 1318 GNUNET_CONTAINER_multihashmap_iterate (out_map, 1319 &free_roi, 1320 NULL); 1321 process_debits (wa->next); 1322 } 1323 1324 1325 /** 1326 * This function is called for all transactions that 1327 * are debited from the exchange's account (outgoing 1328 * transactions). 1329 * 1330 * @param cls `struct WireAccount` with current wire account to process 1331 * @param dhr HTTP response details 1332 */ 1333 static void 1334 history_debit_cb ( 1335 void *cls, 1336 const struct TALER_BANK_DebitHistoryResponse *dhr); 1337 1338 1339 /** 1340 * Task scheduled to begin long-polling on the 1341 * bank transfer. 1342 * 1343 * @param cls a `struct WireAccount *` 1344 */ 1345 static void 1346 dh_long_poll (void *cls) 1347 { 1348 struct WireAccount *wa = cls; 1349 1350 wa->dhh_task = NULL; 1351 wa->dhh_next 1352 = GNUNET_TIME_relative_to_absolute (MIN_LONGPOLL_DELAY); 1353 GNUNET_assert (NULL == wa->dhh); 1354 wa->dhh = TALER_BANK_debit_history ( 1355 ctx, 1356 wa->ai->auth, 1357 wa->wire_off_out, 1358 MAX_PER_TRANSACTION, 1359 MAX_LONGPOLL_DELAY, 1360 &history_debit_cb, 1361 wa); 1362 if (NULL == wa->dhh) 1363 { 1364 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1365 "Failed to start long-polling for bank transaction history for `%s'\n", 1366 wa->ai->section_name); 1367 global_ret = EXIT_FAILURE; 1368 GNUNET_SCHEDULER_shutdown (); 1369 return; 1370 } 1371 } 1372 1373 1374 static void 1375 history_debit_cb ( 1376 void *cls, 1377 const struct TALER_BANK_DebitHistoryResponse *dhr) 1378 { 1379 struct WireAccount *wa = cls; 1380 struct WireTransferOutInfo *roi; 1381 size_t slen; 1382 1383 wa->dhh = NULL; 1384 if ( (MHD_HTTP_OK == dhr->http_status) && 1385 (0 != dhr->details.ok.details_length) ) 1386 { 1387 /* As we got results, we go again *immediately* */ 1388 wa->dhh_next = GNUNET_TIME_UNIT_ZERO_ABS; 1389 } 1390 GNUNET_assert (NULL == wa->dhh_task); 1391 wa->dhh_task 1392 = GNUNET_SCHEDULER_add_at (wa->dhh_next, 1393 &dh_long_poll, 1394 wa); 1395 switch (dhr->http_status) 1396 { 1397 case MHD_HTTP_OK: 1398 for (unsigned int i = 0; i < dhr->details.ok.details_length; i++) 1399 { 1400 const struct TALER_BANK_DebitDetails *dd 1401 = &dhr->details.ok.details[i]; 1402 1403 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1404 "Analyzing bank DEBIT #%llu at %s of %s with WTID %s\n", 1405 (unsigned long long) dd->serial_id, 1406 GNUNET_TIME_timestamp2s (dd->execution_date), 1407 TALER_amount2s (&dd->amount), 1408 TALER_B2S (&dd->wtid)); 1409 wa->wire_off_out = dd->serial_id + 1; 1410 slen = strlen (dd->credit_account_uri.full_payto) + 1; 1411 roi = GNUNET_malloc (sizeof (struct WireTransferOutInfo) 1412 + slen); 1413 GNUNET_CRYPTO_hash (&dd->wtid, 1414 sizeof (dd->wtid), 1415 &roi->subject_hash); 1416 roi->details = *dd; 1417 roi->details.credit_account_uri.full_payto 1418 = (char *) &roi[1]; 1419 GNUNET_memcpy (&roi[1], 1420 dd->credit_account_uri.full_payto, 1421 slen); 1422 if (GNUNET_OK != 1423 GNUNET_CONTAINER_multihashmap_put (out_map, 1424 &roi->subject_hash, 1425 roi, 1426 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) 1427 { 1428 struct TALER_AUDITORDB_WireFormatInconsistency wfi = { 1429 .amount = dd->amount, 1430 .wire_offset = dd->serial_id, 1431 .diagnostic = (char *) "duplicate outgoing wire transfer subject" 1432 }; 1433 enum GNUNET_DB_QueryStatus qs; 1434 1435 qs = TALER_ARL_adb->insert_wire_format_inconsistency ( 1436 TALER_ARL_adb->cls, 1437 &wfi); 1438 if (qs < 0) 1439 { 1440 global_qs = qs; 1441 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1442 commit (qs); 1443 return; 1444 } 1445 TALER_ARL_amount_add (&TALER_ARL_USE_AB ( 1446 wire_debit_duplicate_transfer_subject_total), 1447 &TALER_ARL_USE_AB ( 1448 wire_debit_duplicate_transfer_subject_total), 1449 &dd->amount); 1450 } 1451 } 1452 check_exchange_wire_out (wa); 1453 return; 1454 case MHD_HTTP_NO_CONTENT: 1455 check_exchange_wire_out (wa); 1456 return; 1457 case MHD_HTTP_NOT_FOUND: 1458 if (ignore_account_404) 1459 { 1460 check_exchange_wire_out (wa); 1461 return; 1462 } 1463 break; 1464 default: 1465 break; 1466 } 1467 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1468 "Error fetching debit history of account %s: %u/%u!\n", 1469 wa->ai->section_name, 1470 dhr->http_status, 1471 (unsigned int) dhr->ec); 1472 commit (GNUNET_DB_STATUS_HARD_ERROR); 1473 global_ret = EXIT_FAILURE; 1474 GNUNET_SCHEDULER_shutdown (); 1475 return; 1476 } 1477 1478 1479 static void 1480 process_debits (struct WireAccount *wa) 1481 { 1482 /* skip accounts where DEBIT is not enabled */ 1483 while ( (NULL != wa) && 1484 (! wa->ai->debit_enabled) ) 1485 wa = wa->next; 1486 if (NULL == wa) 1487 { 1488 /* end of iteration */ 1489 commit (GNUNET_DB_STATUS_SUCCESS_ONE_RESULT); 1490 return; 1491 } 1492 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1493 "Checking bank DEBIT records of account `%s'\n", 1494 wa->ai->section_name); 1495 if ( (NULL == wa->dhh) && 1496 (NULL == wa->dhh_task) ) 1497 { 1498 wa->dhh = TALER_BANK_debit_history ( 1499 ctx, 1500 wa->ai->auth, 1501 wa->wire_off_out, 1502 MAX_PER_TRANSACTION, 1503 GNUNET_TIME_UNIT_ZERO, 1504 &history_debit_cb, 1505 wa); 1506 if (NULL == wa->dhh) 1507 { 1508 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1509 "Failed to obtain bank transaction history for `%s'\n", 1510 wa->ai->section_name); 1511 commit (GNUNET_DB_STATUS_HARD_ERROR); 1512 global_ret = EXIT_FAILURE; 1513 GNUNET_SCHEDULER_shutdown (); 1514 return; 1515 } 1516 } 1517 } 1518 1519 1520 /** 1521 * Function called about reserve closing operations the aggregator triggered. 1522 * 1523 * @param cls closure; NULL 1524 * @param rowid row identifier used to uniquely identify the reserve closing operation 1525 * @param execution_date when did we execute the close operation 1526 * @param amount_with_fee how much did we debit the reserve 1527 * @param closing_fee how much did we charge for closing the reserve 1528 * @param reserve_pub public key of the reserve 1529 * @param receiver_account where did we send the funds, in payto://-format 1530 * @param wtid identifier used for the wire transfer 1531 * @param close_request_row which close request triggered the operation? 1532 * 0 if it was a timeout (not used) 1533 * @return #GNUNET_OK to continue to iterate, #GNUNET_SYSERR to stop 1534 */ 1535 static enum GNUNET_GenericReturnValue 1536 reserve_closed_cb ( 1537 void *cls, 1538 uint64_t rowid, 1539 struct GNUNET_TIME_Timestamp execution_date, 1540 const struct TALER_Amount *amount_with_fee, 1541 const struct TALER_Amount *closing_fee, 1542 const struct TALER_ReservePublicKeyP *reserve_pub, 1543 const struct TALER_FullPayto receiver_account, 1544 const struct TALER_WireTransferIdentifierRawP *wtid, 1545 uint64_t close_request_row) 1546 { 1547 struct ReserveClosure *rc; 1548 struct GNUNET_HashCode key; 1549 1550 (void) cls; 1551 (void) close_request_row; 1552 GNUNET_assert (TALER_ARL_USE_PP (wire_reserve_close_id) <= rowid); 1553 TALER_ARL_USE_PP (wire_reserve_close_id) = rowid + 1; 1554 rc = GNUNET_new (struct ReserveClosure); 1555 if (TALER_ARL_SR_INVALID_NEGATIVE == 1556 TALER_ARL_amount_subtract_neg (&rc->amount, 1557 amount_with_fee, 1558 closing_fee)) 1559 { 1560 struct TALER_AUDITORDB_RowInconsistency ri = { 1561 .row_id = rowid, 1562 .row_table 1563 = (char *) "reserves_closures", 1564 .diagnostic 1565 = (char *) "closing fee above reserve balance (and closed anyway)" 1566 }; 1567 enum GNUNET_DB_QueryStatus qs; 1568 1569 qs = TALER_ARL_adb->insert_row_inconsistency ( 1570 TALER_ARL_adb->cls, 1571 &ri); 1572 if (qs < 0) 1573 { 1574 global_qs = qs; 1575 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1576 return GNUNET_OK; 1577 } 1578 GNUNET_free (rc); 1579 return GNUNET_OK; 1580 } 1581 rc->receiver_account.full_payto 1582 = GNUNET_strdup (receiver_account.full_payto); 1583 rc->wtid = *wtid; 1584 rc->execution_date = execution_date; 1585 rc->rowid = rowid; 1586 hash_rc (rc->receiver_account, 1587 wtid, 1588 &key); 1589 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1590 "Discovered reserve closure %llu (%s) over %s benefiting %s\n", 1591 (unsigned long long) rowid, 1592 GNUNET_h2s (&key), 1593 TALER_amount2s (amount_with_fee), 1594 receiver_account.full_payto); 1595 (void) GNUNET_CONTAINER_multihashmap_put ( 1596 reserve_closures, 1597 &key, 1598 rc, 1599 GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE); 1600 return GNUNET_OK; 1601 } 1602 1603 1604 /** 1605 * Start the database transactions and begin the audit. 1606 * 1607 * @return transaction status code 1608 */ 1609 static enum GNUNET_DB_QueryStatus 1610 begin_transaction (void) 1611 { 1612 enum GNUNET_DB_QueryStatus qs; 1613 1614 if (GNUNET_SYSERR == 1615 TALER_ARL_edb->preflight (TALER_ARL_edb->cls)) 1616 { 1617 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1618 "Failed to initialize exchange database connection.\n"); 1619 return GNUNET_DB_STATUS_HARD_ERROR; 1620 } 1621 if (GNUNET_SYSERR == 1622 TALER_ARL_adb->preflight (TALER_ARL_adb->cls)) 1623 { 1624 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1625 "Failed to initialize auditor database session.\n"); 1626 return GNUNET_DB_STATUS_HARD_ERROR; 1627 } 1628 global_qs = GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1629 if (GNUNET_OK != 1630 TALER_ARL_adb->start (TALER_ARL_adb->cls)) 1631 { 1632 GNUNET_break (0); 1633 return GNUNET_DB_STATUS_HARD_ERROR; 1634 } 1635 if (GNUNET_OK != 1636 TALER_ARL_edb->start_read_only (TALER_ARL_edb->cls, 1637 "wire debit auditor")) 1638 { 1639 GNUNET_break (0); 1640 return GNUNET_DB_STATUS_HARD_ERROR; 1641 } 1642 qs = TALER_ARL_adb->get_balance ( 1643 TALER_ARL_adb->cls, 1644 TALER_ARL_GET_AB (total_drained), 1645 TALER_ARL_GET_AB (total_wire_out), 1646 TALER_ARL_GET_AB (total_bad_amount_out_plus), 1647 TALER_ARL_GET_AB (total_bad_amount_out_minus), 1648 TALER_ARL_GET_AB (total_closure_amount_lag), 1649 TALER_ARL_GET_AB (wire_debit_duplicate_transfer_subject_total), 1650 TALER_ARL_GET_AB (total_wire_out), 1651 NULL); 1652 switch (qs) 1653 { 1654 case GNUNET_DB_STATUS_HARD_ERROR: 1655 GNUNET_break (0); 1656 return qs; 1657 case GNUNET_DB_STATUS_SOFT_ERROR: 1658 GNUNET_break (0); 1659 return qs; 1660 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 1661 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 1662 break; 1663 } 1664 for (struct WireAccount *wa = wa_head; 1665 NULL != wa; 1666 wa = wa->next) 1667 { 1668 GNUNET_asprintf (&wa->label_wire_out_serial_id, 1669 "wire-%s-%s", 1670 wa->ai->section_name, 1671 "wire_out_serial_id"); 1672 GNUNET_asprintf (&wa->label_wire_off_out, 1673 "wire-%s-%s", 1674 wa->ai->section_name, 1675 "wire_off_out"); 1676 qs = TALER_ARL_adb->get_auditor_progress ( 1677 TALER_ARL_adb->cls, 1678 wa->label_wire_out_serial_id, 1679 &wa->last_wire_out_serial_id, 1680 wa->label_wire_off_out, 1681 &wa->wire_off_out, 1682 NULL); 1683 if (0 > qs) 1684 { 1685 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1686 return qs; 1687 } 1688 GNUNET_assert (2 == qs); 1689 wa->start_wire_out_serial_id = wa->last_wire_out_serial_id; 1690 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1691 "Resuming account %s debit audit at %llu/%llu\n", 1692 wa->ai->section_name, 1693 (unsigned long long) wa->last_wire_out_serial_id, 1694 (unsigned long long) wa->wire_off_out); 1695 } 1696 qs = TALER_ARL_adb->get_auditor_progress ( 1697 TALER_ARL_adb->cls, 1698 TALER_ARL_GET_PP (wire_reserve_close_id), 1699 NULL); 1700 if (0 > qs) 1701 { 1702 GNUNET_break (GNUNET_DB_STATUS_SOFT_ERROR == qs); 1703 return qs; 1704 } 1705 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS == qs) 1706 { 1707 GNUNET_log (GNUNET_ERROR_TYPE_MESSAGE, 1708 "First analysis of with wire auditor, starting audit from scratch\n"); 1709 } 1710 else 1711 { 1712 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1713 "Resuming wire debit audit at %llu\n", 1714 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1715 } 1716 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1717 "Iterating over reserve closures from %llu\n", 1718 (unsigned long long) TALER_ARL_USE_PP (wire_reserve_close_id)); 1719 qs = TALER_ARL_edb->select_reserve_closed_above_serial_id ( 1720 TALER_ARL_edb->cls, 1721 TALER_ARL_USE_PP (wire_reserve_close_id), 1722 &reserve_closed_cb, 1723 NULL); 1724 if (0 > qs) 1725 { 1726 GNUNET_break (GNUNET_DB_STATUS_HARD_ERROR == qs); 1727 return GNUNET_DB_STATUS_HARD_ERROR; 1728 } 1729 process_debits (wa_head); 1730 return GNUNET_DB_STATUS_SUCCESS_NO_RESULTS; 1731 } 1732 1733 1734 /** 1735 * Function called with information about a wire account. Adds the 1736 * account to our list for processing (if it is enabled and we can 1737 * load the plugin). 1738 * 1739 * @param cls closure, NULL 1740 * @param ai account information 1741 */ 1742 static void 1743 process_account_cb (void *cls, 1744 const struct TALER_EXCHANGEDB_AccountInfo *ai) 1745 { 1746 struct WireAccount *wa; 1747 1748 (void) cls; 1749 if ( (! ai->debit_enabled) && 1750 (! ai->credit_enabled) ) 1751 return; /* not an active exchange account */ 1752 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1753 "Found exchange account `%s'\n", 1754 ai->section_name); 1755 wa = GNUNET_new (struct WireAccount); 1756 wa->ai = ai; 1757 GNUNET_CONTAINER_DLL_insert (wa_head, 1758 wa_tail, 1759 wa); 1760 } 1761 1762 1763 /** 1764 * Function called on events received from Postgres. 1765 * 1766 * @param cls closure, NULL 1767 * @param extra additional event data provided 1768 * @param extra_size number of bytes in @a extra 1769 */ 1770 static void 1771 db_notify (void *cls, 1772 const void *extra, 1773 size_t extra_size) 1774 { 1775 (void) cls; 1776 (void) extra; 1777 (void) extra_size; 1778 1779 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 1780 "Received notification to wake wire helper\n"); 1781 /* If there are accounts we are still processing, abort 1782 the HTTP requests so we can start afresh. */ 1783 for (struct WireAccount *wa = wa_head; 1784 NULL != wa; 1785 wa = wa->next) 1786 { 1787 if (NULL != wa->dhh) 1788 { 1789 TALER_BANK_debit_history_cancel (wa->dhh); 1790 wa->dhh = NULL; 1791 } 1792 check_exchange_wire_out (wa); 1793 } 1794 1795 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1796 begin_transaction ()) 1797 { 1798 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1799 "Audit failed\n"); 1800 GNUNET_break (0); 1801 global_ret = EXIT_FAILURE; 1802 GNUNET_SCHEDULER_shutdown (); 1803 return; 1804 } 1805 } 1806 1807 1808 /** 1809 * Main function that will be run. 1810 * 1811 * @param cls closure 1812 * @param args remaining command-line arguments 1813 * @param cfgfile name of the configuration file used (for saving, can be NULL!) 1814 * @param c configuration 1815 */ 1816 static void 1817 run (void *cls, 1818 char *const *args, 1819 const char *cfgfile, 1820 const struct GNUNET_CONFIGURATION_Handle *c) 1821 { 1822 (void) cls; 1823 (void) args; 1824 (void) cfgfile; 1825 cfg = c; 1826 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 1827 "Launching wire debit auditor\n"); 1828 if (GNUNET_OK != 1829 TALER_ARL_init (c)) 1830 { 1831 global_ret = EXIT_FAILURE; 1832 return; 1833 } 1834 1835 reserve_closures 1836 = GNUNET_CONTAINER_multihashmap_create (1024, 1837 GNUNET_NO); 1838 GNUNET_assert (GNUNET_OK == 1839 TALER_amount_set_zero (TALER_ARL_currency, 1840 &zero)); 1841 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 1842 NULL); 1843 ctx = GNUNET_CURL_init (&GNUNET_CURL_gnunet_scheduler_reschedule, 1844 &rctx); 1845 rctx = GNUNET_CURL_gnunet_rc_create (ctx); 1846 if (NULL == ctx) 1847 { 1848 GNUNET_break (0); 1849 global_ret = EXIT_FAILURE; 1850 return; 1851 } 1852 reserve_closures = GNUNET_CONTAINER_multihashmap_create (1024, 1853 GNUNET_NO); 1854 out_map = GNUNET_CONTAINER_multihashmap_create (1024, 1855 true); 1856 if (GNUNET_OK != 1857 TALER_EXCHANGEDB_load_accounts (TALER_ARL_cfg, 1858 TALER_EXCHANGEDB_ALO_DEBIT 1859 | TALER_EXCHANGEDB_ALO_CREDIT 1860 | TALER_EXCHANGEDB_ALO_AUTHDATA)) 1861 { 1862 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1863 "No bank accounts configured\n"); 1864 global_ret = EXIT_NOTCONFIGURED; 1865 GNUNET_SCHEDULER_shutdown (); 1866 return; 1867 } 1868 TALER_EXCHANGEDB_find_accounts (&process_account_cb, 1869 NULL); 1870 1871 if (0 == test_mode) 1872 { 1873 struct GNUNET_DB_EventHeaderP es = { 1874 .size = htons (sizeof (es)), 1875 .type = htons (TALER_DBEVENT_EXCHANGE_AUDITOR_WAKE_HELPER_WIRE) 1876 }; 1877 1878 eh = TALER_ARL_adb->event_listen (TALER_ARL_adb->cls, 1879 &es, 1880 GNUNET_TIME_UNIT_FOREVER_REL, 1881 &db_notify, 1882 NULL); 1883 GNUNET_assert (NULL != eh); 1884 } 1885 if (GNUNET_DB_STATUS_SUCCESS_NO_RESULTS != 1886 begin_transaction ()) 1887 { 1888 GNUNET_break (0); 1889 global_ret = EXIT_FAILURE; 1890 GNUNET_SCHEDULER_shutdown (); 1891 return; 1892 } 1893 } 1894 1895 1896 /** 1897 * The main function of the wire auditing tool. Checks that 1898 * the exchange's records of wire transfers match that of 1899 * the wire gateway. 1900 * 1901 * @param argc number of arguments from the command line 1902 * @param argv command line arguments 1903 * @return 0 ok, 1 on error 1904 */ 1905 int 1906 main (int argc, 1907 char *const *argv) 1908 { 1909 const struct GNUNET_GETOPT_CommandLineOption options[] = { 1910 GNUNET_GETOPT_option_flag ('i', 1911 "internal", 1912 "perform checks only applicable for exchange-internal audits", 1913 &internal_checks), 1914 GNUNET_GETOPT_option_flag ('I', 1915 "ignore-not-found", 1916 "continue, even if the bank account of the exchange was not found", 1917 &ignore_account_404), 1918 GNUNET_GETOPT_option_flag ('t', 1919 "test", 1920 "run in test mode and exit when idle", 1921 &test_mode), 1922 GNUNET_GETOPT_option_timetravel ('T', 1923 "timetravel"), 1924 GNUNET_GETOPT_OPTION_END 1925 }; 1926 enum GNUNET_GenericReturnValue ret; 1927 1928 ret = GNUNET_PROGRAM_run ( 1929 TALER_AUDITOR_project_data (), 1930 argc, 1931 argv, 1932 "taler-helper-auditor-wire-debit", 1933 gettext_noop ( 1934 "Audit exchange database for consistency with the bank's outgoing wire transfers"), 1935 options, 1936 &run, 1937 NULL); 1938 if (GNUNET_SYSERR == ret) 1939 return EXIT_INVALIDARGUMENT; 1940 if (GNUNET_NO == ret) 1941 return EXIT_SUCCESS; 1942 return global_ret; 1943 } 1944 1945 1946 /* end of taler-helper-auditor-wire-debit.c */