taler-auditor-sync.c (38279B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2020-2022 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file taler-auditor-sync.c 18 * @brief Tool used by the auditor to make a 'safe' copy of the exchanges' database. 19 * @author Christian Grothoff 20 */ 21 #include "platform.h" 22 #include "exchangedb_lib.h" 23 #include "exchange-database/preflight.h" 24 #include "exchange-database/get_exists_aml_officer.h" 25 #include "exchange-database/abort_shard.h" 26 #include "exchange-database/insert_signkey.h" 27 #include "exchange-database/do_aggregate.h" 28 #include "exchange-database/begin_revolving_shard.h" 29 #include "exchange-database/begin_shard.h" 30 #include "exchange-database/update_to_aml_unlocked.h" 31 #include "exchange-database/commit.h" 32 #include "exchange-database/update_shard_progress.h" 33 #include "exchange-database/compute_shard.h" 34 #include "exchange-database/get_count_known_coins.h" 35 #include "exchange-database/insert_aggregation_transient.h" 36 #include "exchange-database/create_tables.h" 37 #include "exchange-database/delete_aggregation_transient.h" 38 #include "exchange-database/delete_shard_locks.h" 39 #include "exchange-database/disable_rules.h" 40 #include "exchange-database/do_check_deposit_idempotent.h" 41 #include "exchange-database/do_deposit.h" 42 #include "exchange-database/do_purse_delete.h" 43 #include "exchange-database/do_purse_deposit.h" 44 #include "exchange-database/do_purse_merge.h" 45 #include "exchange-database/do_recoup.h" 46 #include "exchange-database/do_recoup_refresh.h" 47 #include "exchange-database/do_refresh.h" 48 #include "exchange-database/do_refund.h" 49 #include "exchange-database/do_reserve_open.h" 50 #include "exchange-database/do_reserve_purse.h" 51 #include "exchange-database/do_withdraw.h" 52 #include "exchange-database/do_drain_kyc_alert.h" 53 #include "exchange-database/drop_tables.h" 54 #include "exchange-database/enable_rules.h" 55 #include "exchange-database/event_listen_cancel.h" 56 #include "exchange-database/event_listen.h" 57 #include "exchange-database/do_expire_purse.h" 58 #include "exchange-database/get_aggregation_transient_by_normalized_payto.h" 59 #include "exchange-database/gc.h" 60 #include "exchange-database/get_coin_denomination.h" 61 #include "exchange-database/get_coin_transactions.h" 62 #include "exchange-database/get_denomination_by_serial.h" 63 #include "exchange-database/get_denomination_info.h" 64 #include "exchange-database/get_denomination_revocation.h" 65 #include "exchange-database/get_profit_drain.h" 66 #include "exchange-database/iterate_expired_reserves.h" 67 #include "exchange-database/get_global_fee.h" 68 #include "exchange-database/iterate_global_fees.h" 69 #include "exchange-database/get_known_coin.h" 70 #include "exchange-database/get_kyc_rules.h" 71 #include "exchange-database/get_pending_legitimization_process.h" 72 #include "exchange-database/get_purse_deposit.h" 73 #include "exchange-database/get_purse_request.h" 74 #include "exchange-database/get_ready_deposit.h" 75 #include "exchange-database/get_refresh.h" 76 #include "exchange-database/get_reserve_balance.h" 77 #include "exchange-database/get_reserve_history.h" 78 #include "exchange-database/get_signature_for_known_coin.h" 79 #include "exchange-database/iterate_unfinished_close_requests.h" 80 #include "exchange-database/iterate_wire_accounts.h" 81 #include "exchange-database/get_wire_fee.h" 82 #include "exchange-database/iterate_wire_fees.h" 83 #include "exchange-database/get_wire_hash_for_contract.h" 84 #include "exchange-database/get_withdraw.h" 85 #include "exchange-database/get_exists_deposit.h" 86 #include "exchange-database/inject_auditor_triggers.h" 87 #include "exchange-database/insert_active_legitimization_measure.h" 88 #include "exchange-database/insert_aml_decision.h" 89 #include "exchange-database/insert_aml_officer.h" 90 #include "exchange-database/insert_aml_program_failure.h" 91 #include "exchange-database/insert_auditor_denom_sig.h" 92 #include "exchange-database/insert_auditor.h" 93 #include "exchange-database/insert_close_request.h" 94 #include "exchange-database/insert_contract.h" 95 #include "exchange-database/insert_denomination_info.h" 96 #include "exchange-database/insert_denomination_revocation.h" 97 #include "exchange-database/insert_profit_drain.h" 98 #include "exchange-database/insert_global_fee.h" 99 #include "exchange-database/insert_kyc_failure.h" 100 #include "exchange-database/insert_legitimization_process.h" 101 #include "exchange-database/insert_partner.h" 102 #include "exchange-database/insert_purse_request.h" 103 struct InsertContext; 104 #define TALER_EXCHANGEDB_REPLICATION_RESULT_CLOSURE struct InsertContext 105 #include "exchange-database/insert_records_by_table.h" 106 #include "exchange-database/insert_reserve_closed.h" 107 #include "exchange-database/insert_reserve_open_deposit.h" 108 #include "exchange-database/insert_sanction_list_hit.h" 109 #include "exchange-database/insert_signkey_revocation.h" 110 #include "exchange-database/insert_successor_measure.h" 111 #include "exchange-database/insert_wire_fee.h" 112 #include "exchange-database/insert_wire.h" 113 #include "exchange-database/iterate_active_auditors.h" 114 #include "exchange-database/iterate_active_signkeys.h" 115 #include "exchange-database/iterate_auditor_denominations.h" 116 #include "exchange-database/iterate_denomination_info.h" 117 #include "exchange-database/iterate_denominations.h" 118 #include "exchange-database/iterate_kyc_references.h" 119 #include "exchange-database/iterate_reserve_close_info.h" 120 #include "exchange-database/get_kyc_provider_account.h" 121 #include "exchange-database/get_active_legitimization.h" 122 #include "exchange-database/get_aml_file_number.h" 123 #include "exchange-database/iterate_aml_history.h" 124 #include "exchange-database/get_aml_officer.h" 125 #include "exchange-database/get_auditor_status.h" 126 #include "exchange-database/get_auditor_timestamp.h" 127 #include "exchange-database/get_completed_legitimization.h" 128 #include "exchange-database/get_denomination_meta.h" 129 #include "exchange-database/get_global_fee_by_time.h" 130 #include "exchange-database/get_h_payto_by_access_token.h" 131 #include "exchange-database/iterate_kyc_history.h" 132 #include "exchange-database/get_legitimization_process_by_account.h" 133 #include "exchange-database/get_legitimization_requirement_by_row.h" 134 #include "exchange-database/get_kyc_status_by_token.h" 135 #include "exchange-database/get_pending_legitimization.h" 136 #include "exchange-database/iterate_records_by_table.h" 137 #include "exchange-database/get_rules_by_access_token.h" 138 #include "exchange-database/get_serial_by_table.h" 139 #include "exchange-database/do_sync_refresh.h" 140 #include "exchange-database/get_signkey.h" 141 #include "exchange-database/get_signkey_revocation.h" 142 #include "exchange-database/get_transfer_by_deposit.h" 143 #include "exchange-database/get_wire_fee_by_time.h" 144 #include "exchange-database/get_wire_timestamp.h" 145 #include "exchange-database/iterate_wire_transfers.h" 146 #include "exchange-database/update_to_refresh_revealed.h" 147 #include "exchange-database/do_insert_kyc_attributes.h" 148 #include "exchange-database/preflight.h" 149 #include "exchange-database/get_pending_profit_drain.h" 150 #include "exchange-database/update_to_profit_drain_finished.h" 151 #include "exchange-database/release_revolving_shard.h" 152 #include "exchange-database/get_reserve.h" 153 #include "exchange-database/get_reserve_origin.h" 154 #include "exchange-database/do_import_credits.h" 155 #include "exchange-database/update_reserve.h" 156 #include "exchange-database/rollback.h" 157 #include "exchange-database/iterate_account_merges_above_serial_id.h" 158 #include "exchange-database/iterate_aggregation_amounts_for_kyc_check.h" 159 #include "exchange-database/iterate_aggregations_above_serial_id.h" 160 #include "exchange-database/get_aggregation_transient.h" 161 #include "exchange-database/iterate_all_purse_decisions_above_serial_id.h" 162 #include "exchange-database/iterate_all_purse_deletions_above_serial_id.h" 163 #include "exchange-database/iterate_aml_attributes.h" 164 #include "exchange-database/iterate_aml_decisions.h" 165 #include "exchange-database/iterate_aml_measures.h" 166 #include "exchange-database/iterate_aml_statistics.h" 167 #include "exchange-database/get_auditor_denom_sig.h" 168 #include "exchange-database/iterate_batch_deposits_missing_wire.h" 169 #include "exchange-database/iterate_coin_deposits_above_serial_id.h" 170 #include "exchange-database/get_contract_by_purse.h" 171 #include "exchange-database/get_contract.h" 172 #include "exchange-database/iterate_deposit_amounts_for_kyc_check.h" 173 #include "exchange-database/iterate_exchange_credit_transfers.h" 174 #include "exchange-database/iterate_exchange_debit_transfers.h" 175 #include "exchange-database/iterate_exchange_kycauth_transfers.h" 176 #include "exchange-database/iterate_kyc_accounts.h" 177 #include "exchange-database/iterate_kyc_attributes.h" 178 #include "exchange-database/iterate_merge_amounts_for_kyc_check.h" 179 #include "exchange-database/get_purse_by_merge_pub.h" 180 #include "exchange-database/iterate_purse_decisions_above_serial_id.h" 181 #include "exchange-database/iterate_purse_deposits_above_serial_id.h" 182 #include "exchange-database/iterate_purse_deposits_by_purse.h" 183 #include "exchange-database/get_purse.h" 184 #include "exchange-database/get_purse_merge.h" 185 #include "exchange-database/iterate_purse_merges_above_serial_id.h" 186 #include "exchange-database/iterate_purse_requests_above_serial_id.h" 187 #include "exchange-database/iterate_recoups_above_serial_id.h" 188 #include "exchange-database/iterate_recoup_refreshes_above_serial_id.h" 189 #include "exchange-database/iterate_refreshes_above_serial_id.h" 190 #include "exchange-database/iterate_refunds_above_serial_id.h" 191 #include "exchange-database/iterate_refunds_by_coin.h" 192 #include "exchange-database/iterate_reserve_closed_above_serial_id.h" 193 #include "exchange-database/get_reserve_close_info.h" 194 #include "exchange-database/iterate_reserve_open_above_serial_id.h" 195 #include "exchange-database/iterate_reserves_in_above_serial_id.h" 196 #include "exchange-database/iterate_wire_outs_above_serial_id_by_account.h" 197 #include "exchange-database/iterate_wire_outs_above_serial_id.h" 198 #include "exchange-database/iterate_withdrawals_above_serial_id.h" 199 #include "exchange-database/iterate_withdraw_amounts_for_kyc_check.h" 200 #include "exchange-database/update_to_aml_locked.h" 201 #include "exchange-database/start_deferred_wire_out.h" 202 #include "exchange-database/start.h" 203 #include "exchange-database/start_read_committed.h" 204 #include "exchange-database/start_read_only.h" 205 #include "exchange-database/insert_wire_out.h" 206 #include "exchange-database/get_exists_aml_officer.h" 207 #include "exchange-database/do_trigger_kyc_rule_for_account.h" 208 #include "exchange-database/update_aggregation_transient.h" 209 #include "exchange-database/update_auditor.h" 210 #include "exchange-database/update_legitimization_process_by_row.h" 211 #include "exchange-database/update_wire.h" 212 #include "exchange-database/iterate_prewires.h" 213 #include "exchange-database/insert_prewire.h" 214 #include "exchange-database/update_to_prewire_failed.h" 215 #include "exchange-database/update_to_prewire_finished.h" 216 217 218 /** 219 * Handle to access the exchange's source database. 220 */ 221 static struct TALER_EXCHANGEDB_PostgresContext *src; 222 223 /** 224 * Handle to access the exchange's destination database. 225 */ 226 static struct TALER_EXCHANGEDB_PostgresContext *dst; 227 228 /** 229 * Return value from #main(). 230 */ 231 static int global_ret; 232 233 /** 234 * Main task to do synchronization. 235 */ 236 static struct GNUNET_SCHEDULER_Task *sync_task; 237 238 /** 239 * What is our target transaction size (number of records)? 240 */ 241 static unsigned int transaction_size = 512; 242 243 /** 244 * Number of records copied in this transaction. 245 */ 246 static unsigned long long actual_size; 247 248 /** 249 * Terminate once synchronization is achieved. 250 */ 251 static int exit_if_synced; 252 253 254 /** 255 * Highest value in `enum TALER_EXCHANGEDB_ReplicatedTable`. Update 256 * together with #rt_name() and #tables when the exchange gains another 257 * replicated table. 258 */ 259 #define RT_MAX TALER_EXCHANGEDB_RT_KYCAUTHS_IN 260 261 262 /** 263 * Return the name of the database table @a rt lives in. 264 * 265 * The switch is exhaustive and deliberately has no @e default case: when 266 * a table is added to `enum TALER_EXCHANGEDB_ReplicatedTable`, the 267 * compiler points at this function (-Wswitch), which is the reminder 268 * that #RT_MAX and #tables below must be extended as well. #check_tables() 269 * catches it at runtime if they were not. 270 * 271 * @param rt table to name 272 * @return name of the table in the database 273 */ 274 static const char * 275 rt_name (enum TALER_EXCHANGEDB_ReplicatedTable rt) 276 { 277 switch (rt) 278 { 279 case TALER_EXCHANGEDB_RT_DENOMINATIONS: 280 return "denominations"; 281 case TALER_EXCHANGEDB_RT_DENOMINATION_REVOCATIONS: 282 return "denomination_revocations"; 283 case TALER_EXCHANGEDB_RT_KYC_TARGETS: 284 return "kyc_targets"; 285 case TALER_EXCHANGEDB_RT_WIRE_TARGETS: 286 return "wire_targets"; 287 case TALER_EXCHANGEDB_RT_RESERVES: 288 return "reserves"; 289 case TALER_EXCHANGEDB_RT_RESERVES_IN: 290 return "reserves_in"; 291 case TALER_EXCHANGEDB_RT_RESERVES_CLOSE: 292 return "reserves_close"; 293 case TALER_EXCHANGEDB_RT_RESERVES_OPEN_REQUESTS: 294 return "reserves_open_requests"; 295 case TALER_EXCHANGEDB_RT_RESERVES_OPEN_DEPOSITS: 296 return "reserves_open_deposits"; 297 case TALER_EXCHANGEDB_RT_AUDITORS: 298 return "auditors"; 299 case TALER_EXCHANGEDB_RT_AUDITOR_DENOM_SIGS: 300 return "auditor_denom_sigs"; 301 case TALER_EXCHANGEDB_RT_EXCHANGE_SIGN_KEYS: 302 return "exchange_sign_keys"; 303 case TALER_EXCHANGEDB_RT_SIGNKEY_REVOCATIONS: 304 return "signkey_revocations"; 305 case TALER_EXCHANGEDB_RT_KNOWN_COINS: 306 return "known_coins"; 307 case TALER_EXCHANGEDB_RT_REFRESH: 308 return "refresh"; 309 case TALER_EXCHANGEDB_RT_BATCH_DEPOSITS: 310 return "batch_deposits"; 311 case TALER_EXCHANGEDB_RT_COIN_DEPOSITS: 312 return "coin_deposits"; 313 case TALER_EXCHANGEDB_RT_REFUNDS: 314 return "refunds"; 315 case TALER_EXCHANGEDB_RT_WIRE_OUT: 316 return "wire_out"; 317 case TALER_EXCHANGEDB_RT_AGGREGATION_TRACKING: 318 return "aggregation_tracking"; 319 case TALER_EXCHANGEDB_RT_AGGREGATION_DEFERRALS: 320 return "aggregation_deferrals"; 321 case TALER_EXCHANGEDB_RT_WIRE_FEE: 322 return "wire_fee"; 323 case TALER_EXCHANGEDB_RT_GLOBAL_FEE: 324 return "global_fee"; 325 case TALER_EXCHANGEDB_RT_RECOUP: 326 return "recoup"; 327 case TALER_EXCHANGEDB_RT_RECOUP_REFRESH: 328 return "recoup_refresh"; 329 case TALER_EXCHANGEDB_RT_PURSE_REQUESTS: 330 return "purse_requests"; 331 case TALER_EXCHANGEDB_RT_PURSE_DECISION: 332 return "purse_decision"; 333 case TALER_EXCHANGEDB_RT_PURSE_MERGES: 334 return "purse_merges"; 335 case TALER_EXCHANGEDB_RT_PURSE_DEPOSITS: 336 return "purse_deposits"; 337 case TALER_EXCHANGEDB_RT_ACCOUNT_MERGES: 338 return "account_merges"; 339 case TALER_EXCHANGEDB_RT_HISTORY_REQUESTS: 340 return "history_requests"; 341 case TALER_EXCHANGEDB_RT_CLOSE_REQUESTS: 342 return "close_requests"; 343 case TALER_EXCHANGEDB_RT_WADS_OUT: 344 return "wads_out"; 345 case TALER_EXCHANGEDB_RT_WADS_OUT_ENTRIES: 346 return "wad_out_entries"; 347 case TALER_EXCHANGEDB_RT_WADS_IN: 348 return "wads_in"; 349 case TALER_EXCHANGEDB_RT_WADS_IN_ENTRIES: 350 return "wad_in_entries"; 351 case TALER_EXCHANGEDB_RT_PROFIT_DRAINS: 352 return "profit_drains"; 353 case TALER_EXCHANGEDB_RT_AML_STAFF: 354 return "aml_staff"; 355 case TALER_EXCHANGEDB_RT_PURSE_DELETION: 356 return "purse_deletion"; 357 case TALER_EXCHANGEDB_RT_WITHDRAW: 358 return "withdraw"; 359 case TALER_EXCHANGEDB_RT_LEGITIMIZATION_MEASURES: 360 return "legitimization_measures"; 361 case TALER_EXCHANGEDB_RT_LEGITIMIZATION_OUTCOMES: 362 return "legitimization_outcomes"; 363 case TALER_EXCHANGEDB_RT_LEGITIMIZATION_PROCESSES: 364 return "legitimization_processes"; 365 case TALER_EXCHANGEDB_RT_KYC_ATTRIBUTES: 366 return "kyc_attributes"; 367 case TALER_EXCHANGEDB_RT_AML_HISTORY: 368 return "aml_history"; 369 case TALER_EXCHANGEDB_RT_KYC_EVENTS: 370 return "kyc_events"; 371 case TALER_EXCHANGEDB_RT_KYCAUTHS_IN: 372 return "kycauths_in"; 373 } 374 GNUNET_break (0); 375 return "<unknown>"; 376 } 377 378 379 /** 380 * Information we track per replicated table. 381 */ 382 struct Table 383 { 384 /** 385 * Which table is this record about? 386 */ 387 enum TALER_EXCHANGEDB_ReplicatedTable rt; 388 389 /** 390 * Up to which record is the destination table synchronized. 391 */ 392 uint64_t start_serial; 393 394 /** 395 * Highest serial in the source table. 396 */ 397 uint64_t end_serial; 398 399 /** 400 * Marker for the end of the list of #tables. 401 */ 402 bool end; 403 }; 404 405 406 /** 407 * Information about replicated tables. 408 * 409 * The order matters: a table must be listed after every table its 410 * foreign keys point to, as the destination database enforces those 411 * constraints just like the source does. Must cover every value of 412 * `enum TALER_EXCHANGEDB_ReplicatedTable`, which #check_tables() verifies 413 * at startup. 414 */ 415 static struct Table tables[] = { 416 { .rt = TALER_EXCHANGEDB_RT_DENOMINATIONS}, 417 { .rt = TALER_EXCHANGEDB_RT_DENOMINATION_REVOCATIONS}, 418 { .rt = TALER_EXCHANGEDB_RT_KYC_TARGETS}, 419 { .rt = TALER_EXCHANGEDB_RT_WIRE_TARGETS}, 420 { .rt = TALER_EXCHANGEDB_RT_LEGITIMIZATION_MEASURES}, 421 { .rt = TALER_EXCHANGEDB_RT_LEGITIMIZATION_OUTCOMES}, 422 { .rt = TALER_EXCHANGEDB_RT_LEGITIMIZATION_PROCESSES}, 423 /* kyc_attributes.legitimization_serial references 424 legitimization_processes */ 425 { .rt = TALER_EXCHANGEDB_RT_KYC_ATTRIBUTES}, 426 { .rt = TALER_EXCHANGEDB_RT_AML_STAFF}, 427 /* aml_history.outcome_serial_id references legitimization_outcomes, 428 aml_history.kyc_attributes_serial_id references kyc_attributes */ 429 { .rt = TALER_EXCHANGEDB_RT_AML_HISTORY}, 430 { .rt = TALER_EXCHANGEDB_RT_KYC_EVENTS}, 431 { .rt = TALER_EXCHANGEDB_RT_RESERVES}, 432 { .rt = TALER_EXCHANGEDB_RT_RESERVES_IN}, 433 { .rt = TALER_EXCHANGEDB_RT_KYCAUTHS_IN}, 434 { .rt = TALER_EXCHANGEDB_RT_RESERVES_CLOSE}, 435 { .rt = TALER_EXCHANGEDB_RT_RESERVES_OPEN_REQUESTS}, 436 { .rt = TALER_EXCHANGEDB_RT_RESERVES_OPEN_DEPOSITS}, 437 { .rt = TALER_EXCHANGEDB_RT_WITHDRAW}, 438 { .rt = TALER_EXCHANGEDB_RT_AUDITORS}, 439 { .rt = TALER_EXCHANGEDB_RT_AUDITOR_DENOM_SIGS}, 440 { .rt = TALER_EXCHANGEDB_RT_EXCHANGE_SIGN_KEYS}, 441 { .rt = TALER_EXCHANGEDB_RT_SIGNKEY_REVOCATIONS}, 442 { .rt = TALER_EXCHANGEDB_RT_KNOWN_COINS}, 443 { .rt = TALER_EXCHANGEDB_RT_REFRESH}, 444 { .rt = TALER_EXCHANGEDB_RT_BATCH_DEPOSITS}, 445 { .rt = TALER_EXCHANGEDB_RT_COIN_DEPOSITS}, 446 { .rt = TALER_EXCHANGEDB_RT_REFUNDS}, 447 { .rt = TALER_EXCHANGEDB_RT_WIRE_OUT}, 448 { .rt = TALER_EXCHANGEDB_RT_AGGREGATION_TRACKING}, 449 /* after batch_deposits: the foreign key that ties a deferral's lifetime to 450 the deposits it is about is enforced here too */ 451 { .rt = TALER_EXCHANGEDB_RT_AGGREGATION_DEFERRALS}, 452 { .rt = TALER_EXCHANGEDB_RT_WIRE_FEE}, 453 { .rt = TALER_EXCHANGEDB_RT_GLOBAL_FEE}, 454 { .rt = TALER_EXCHANGEDB_RT_RECOUP}, 455 { .rt = TALER_EXCHANGEDB_RT_RECOUP_REFRESH }, 456 { .rt = TALER_EXCHANGEDB_RT_PURSE_REQUESTS}, 457 { .rt = TALER_EXCHANGEDB_RT_PURSE_DECISION}, 458 { .rt = TALER_EXCHANGEDB_RT_PURSE_MERGES}, 459 { .rt = TALER_EXCHANGEDB_RT_PURSE_DEPOSITS}, 460 { .rt = TALER_EXCHANGEDB_RT_PURSE_DELETION}, 461 { .rt = TALER_EXCHANGEDB_RT_ACCOUNT_MERGES}, 462 { .rt = TALER_EXCHANGEDB_RT_HISTORY_REQUESTS}, 463 { .rt = TALER_EXCHANGEDB_RT_CLOSE_REQUESTS}, 464 { .rt = TALER_EXCHANGEDB_RT_WADS_OUT}, 465 { .rt = TALER_EXCHANGEDB_RT_WADS_OUT_ENTRIES}, 466 { .rt = TALER_EXCHANGEDB_RT_WADS_IN}, 467 { .rt = TALER_EXCHANGEDB_RT_WADS_IN_ENTRIES}, 468 { .rt = TALER_EXCHANGEDB_RT_PROFIT_DRAINS}, 469 { .end = true } 470 }; 471 472 473 /** 474 * Check that #tables lists every replicated table exactly once. A table 475 * missing here is not an error the operator could ever notice: the sync 476 * would simply run to completion with that table left empty. 477 * 478 * @return #GNUNET_OK if #tables is complete 479 */ 480 static enum GNUNET_GenericReturnValue 481 check_tables (void) 482 { 483 bool seen[RT_MAX + 1] = { false }; 484 enum GNUNET_GenericReturnValue ret = GNUNET_OK; 485 486 for (unsigned int i = 0; ! tables[i].end; i++) 487 { 488 enum TALER_EXCHANGEDB_ReplicatedTable rt = tables[i].rt; 489 490 if (seen[rt]) 491 { 492 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 493 "Table `%s' is listed for replication more than once\n", 494 rt_name (rt)); 495 ret = GNUNET_SYSERR; 496 } 497 seen[rt] = true; 498 } 499 for (unsigned int i = 0; i <= RT_MAX; i++) 500 { 501 if (seen[i]) 502 continue; 503 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 504 "Table `%s' would not be replicated\n", 505 rt_name ((enum TALER_EXCHANGEDB_ReplicatedTable) i)); 506 ret = GNUNET_SYSERR; 507 } 508 return ret; 509 } 510 511 512 /** 513 * Closure for #do_insert. 514 */ 515 struct InsertContext 516 { 517 /** 518 * Table we are replicating. 519 */ 520 struct Table *table; 521 522 /** 523 * Set to error if insertion created an error. 524 */ 525 enum GNUNET_DB_QueryStatus qs; 526 }; 527 528 529 /** 530 * Function called on data to replicate in the auditor's database. 531 * 532 * @param ctx closure, a `struct InsertContext` 533 * @param td record from an exchange table 534 * @return #GNUNET_OK to continue to iterate, 535 * #GNUNET_SYSERR to fail with an error 536 */ 537 static enum GNUNET_GenericReturnValue 538 do_insert (struct InsertContext *ctx, 539 const struct TALER_EXCHANGEDB_TableData *td) 540 { 541 enum GNUNET_DB_QueryStatus qs; 542 543 if (0 >= ctx->qs) 544 return GNUNET_SYSERR; 545 qs = TALER_EXCHANGEDB_insert_records_by_table (dst, 546 td); 547 if (0 >= qs) 548 { 549 switch (qs) 550 { 551 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 552 GNUNET_assert (0); 553 break; 554 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 555 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 556 "Failed to insert record into table `%s': no change\n", 557 rt_name (td->table)); 558 break; 559 case GNUNET_DB_STATUS_SOFT_ERROR: 560 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 561 "Serialization error inserting record into table `%s' (will retry)\n", 562 rt_name (td->table)); 563 break; 564 case GNUNET_DB_STATUS_HARD_ERROR: 565 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 566 "Failed to insert record into table `%s': hard error\n", 567 rt_name (td->table)); 568 break; 569 } 570 ctx->qs = qs; 571 return GNUNET_SYSERR; 572 } 573 actual_size++; 574 ctx->table->start_serial = td->serial; 575 return GNUNET_OK; 576 } 577 578 579 /** 580 * Determine the last serial number of each table in @a db. 581 * 582 * Failing to determine a serial must abort the pass: the previous 583 * value of the field is stale (#tables is file-scope), and continuing 584 * with it would silently skip records or entire tables. 585 * 586 * @param db database to query 587 * @param source true to store the result in the `end_serial` of each 588 * table (data source), false for `start_serial` (destination) 589 * @return #GNUNET_OK on success, #GNUNET_SYSERR to rollback 590 */ 591 static enum GNUNET_GenericReturnValue 592 lookup_serials (struct TALER_EXCHANGEDB_PostgresContext *db, 593 bool source) 594 { 595 for (unsigned int i = 0; ! tables[i].end; i++) 596 { 597 uint64_t *serial = source 598 ? &tables[i].end_serial 599 : &tables[i].start_serial; 600 enum GNUNET_DB_QueryStatus qs; 601 602 qs = TALER_EXCHANGEDB_get_serial_by_table (db, 603 tables[i].rt, 604 serial); 605 switch (qs) 606 { 607 case GNUNET_DB_STATUS_HARD_ERROR: 608 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 609 "Failed to determine last serial of table `%s': hard error\n", 610 rt_name (tables[i].rt)); 611 global_ret = EXIT_FAILURE; 612 return GNUNET_SYSERR; 613 case GNUNET_DB_STATUS_SOFT_ERROR: 614 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 615 "Serialization error determining last serial of table `%s' (will retry)\n", 616 rt_name (tables[i].rt)); 617 return GNUNET_SYSERR; 618 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 619 /* Table is empty; the statement did not write *serial. */ 620 *serial = 0; 621 break; 622 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 623 break; 624 } 625 } 626 return GNUNET_OK; 627 } 628 629 630 /** 631 * Run one replication transaction. 632 * 633 * @return #GNUNET_OK on success, #GNUNET_SYSERR to rollback 634 */ 635 static enum GNUNET_GenericReturnValue 636 transact (void) 637 { 638 struct InsertContext ctx = { 639 .qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT 640 }; 641 642 if (0 > 643 TALER_EXCHANGEDB_start (src, 644 "lookup src serials")) 645 return GNUNET_SYSERR; 646 if (GNUNET_OK != 647 lookup_serials (src, 648 true)) 649 return GNUNET_SYSERR; 650 TALER_EXCHANGEDB_rollback (src); 651 if (GNUNET_OK != 652 TALER_EXCHANGEDB_start (dst, 653 "lookup dst serials")) 654 return GNUNET_SYSERR; 655 if (GNUNET_OK != 656 lookup_serials (dst, 657 false)) 658 return GNUNET_SYSERR; 659 TALER_EXCHANGEDB_rollback (dst); 660 for (unsigned int i = 0; ! tables[i].end; i++) 661 { 662 struct Table *table = &tables[i]; 663 664 if (table->start_serial == table->end_serial) 665 continue; 666 if (table->start_serial > table->end_serial) 667 { 668 /* The destination is ahead of the source: either we are copying 669 from the wrong database, or the source was restored from an old 670 backup. Either way, continuing would produce a bogus copy. */ 671 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 672 "Destination is ahead of source for table `%s' (%llu > %llu); aborting\n", 673 rt_name (table->rt), 674 (unsigned long long) table->start_serial, 675 (unsigned long long) table->end_serial); 676 global_ret = EXIT_FAILURE; 677 return GNUNET_SYSERR; 678 } 679 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 680 "Replicating table `%s' from %llu to %llu\n", 681 rt_name (table->rt), 682 (unsigned long long) table->start_serial, 683 (unsigned long long) table->end_serial); 684 ctx.table = table; 685 while (table->start_serial < table->end_serial) 686 { 687 enum GNUNET_DB_QueryStatus qs; 688 689 if (GNUNET_OK != 690 TALER_EXCHANGEDB_start (src, 691 "copy table (src)")) 692 return GNUNET_SYSERR; 693 if (GNUNET_OK != 694 TALER_EXCHANGEDB_start (dst, 695 "copy table (dst)")) 696 return GNUNET_SYSERR; 697 qs = TALER_EXCHANGEDB_iterate_records_by_table (src, 698 table->rt, 699 table->start_serial, 700 &do_insert, 701 &ctx); 702 if (ctx.qs < 0) 703 qs = ctx.qs; 704 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 705 { 706 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 707 "Failed to lookup records from table `%s': hard error\n", 708 rt_name (table->rt)); 709 global_ret = EXIT_FAILURE; 710 return GNUNET_SYSERR; 711 } 712 if (GNUNET_DB_STATUS_SOFT_ERROR == qs) 713 { 714 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 715 "Serialization error looking up records from table `%s' (will retry)\n", 716 rt_name (table->rt)); 717 return GNUNET_SYSERR; /* will retry */ 718 } 719 if (0 == qs) 720 { 721 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 722 "Failed to lookup records from table `%s': no results\n", 723 rt_name (table->rt)); 724 GNUNET_break (0); /* should be impossible */ 725 global_ret = EXIT_FAILURE; 726 return GNUNET_SYSERR; 727 } 728 if (0 == ctx.qs) 729 return GNUNET_SYSERR; /* insertion failed, maybe record existed? try again */ 730 TALER_EXCHANGEDB_rollback (src); 731 qs = TALER_EXCHANGEDB_commit (dst); 732 if (GNUNET_DB_STATUS_SOFT_ERROR == qs) 733 { 734 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 735 "Serialization error committing transaction on table `%s' (will retry)\n", 736 rt_name (table->rt)); 737 /* The rows of the failed commit are gone, but start_serial was 738 already advanced past them by do_insert(); continuing here 739 would leave a permanent hole. Restart the pass instead, which 740 re-reads the serials actually present in the destination. */ 741 return GNUNET_SYSERR; 742 } 743 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 744 { 745 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 746 "Hard error committing transaction on table `%s'\n", 747 rt_name (table->rt)); 748 global_ret = EXIT_FAILURE; 749 return GNUNET_SYSERR; 750 } 751 } 752 } 753 { 754 uint64_t after = 0; 755 756 do { 757 enum GNUNET_DB_QueryStatus qs; 758 759 qs = TALER_EXCHANGEDB_do_sync_refresh (src, 760 dst, 761 GNUNET_MAX (1, transaction_size), 762 &after); 763 if (qs < 0) 764 { 765 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 766 global_ret = EXIT_FAILURE; 767 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 768 "Failed to synchronize refresh metadata (%d)\n", 769 (int) qs); 770 return GNUNET_SYSERR; 771 } 772 actual_size += qs; 773 } while (0 != after); 774 } 775 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 776 "Sync pass completed successfully with %llu updates\n", 777 actual_size); 778 return GNUNET_OK; 779 } 780 781 782 /** 783 * Task to do the actual synchronization work. 784 * 785 * @param cls NULL, unused 786 */ 787 static void 788 do_sync (void *cls) 789 { 790 static struct GNUNET_TIME_Relative delay; 791 enum GNUNET_GenericReturnValue ret; 792 793 (void) cls; 794 sync_task = NULL; 795 actual_size = 0; 796 if (GNUNET_SYSERR == 797 TALER_EXCHANGEDB_preflight (src)) 798 { 799 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 800 "Failed to begin transaction with data source. Exiting\n"); 801 global_ret = EXIT_FAILURE; 802 return; 803 } 804 if (GNUNET_SYSERR == 805 TALER_EXCHANGEDB_preflight (dst)) 806 { 807 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 808 "Failed to begin transaction with data destination. Exiting\n"); 809 global_ret = EXIT_FAILURE; 810 return; 811 } 812 ret = transact (); 813 if (GNUNET_OK != ret) 814 { 815 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 816 "Transaction failed, rolling back\n"); 817 TALER_EXCHANGEDB_rollback (src); 818 TALER_EXCHANGEDB_rollback (dst); 819 } 820 if (0 != global_ret) 821 { 822 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 823 "Transaction failed permanently, exiting\n"); 824 return; 825 } 826 /* Only a pass that ran to completion proves the databases agree: 827 a rolled-back pass copies nothing, and reporting that as "in sync" 828 would let -t exit successfully having replicated nothing. */ 829 if ( (GNUNET_OK == ret) && 830 (0 == actual_size) && 831 (exit_if_synced) ) 832 { 833 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 834 "Databases are synchronized. Exiting\n"); 835 return; 836 } 837 if (actual_size < transaction_size / 2) 838 { 839 delay = GNUNET_TIME_STD_BACKOFF (delay); 840 } 841 else if (actual_size >= transaction_size) 842 { 843 delay = GNUNET_TIME_UNIT_ZERO; 844 } 845 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 846 "Next sync pass in %s\n", 847 GNUNET_STRINGS_relative_time_to_string (delay, 848 GNUNET_YES)); 849 sync_task = GNUNET_SCHEDULER_add_delayed (delay, 850 &do_sync, 851 NULL); 852 } 853 854 855 /** 856 * Set an option of type 'char *' from the command line with 857 * filename expansion a la #GNUNET_STRINGS_filename_expand(). 858 * 859 * @param ctx command line processing context 860 * @param scls additional closure (will point to the `char *`, 861 * which will be allocated) 862 * @param option name of the option 863 * @param value actual value of the option (a string) 864 * @return #GNUNET_OK 865 */ 866 static enum GNUNET_GenericReturnValue 867 set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx, 868 void *scls, 869 const char *option, 870 const char *value) 871 { 872 char **val = scls; 873 874 (void) ctx; 875 (void) option; 876 GNUNET_assert (NULL != value); 877 GNUNET_free (*val); 878 *val = GNUNET_STRINGS_filename_expand (value); 879 return GNUNET_OK; 880 } 881 882 883 /** 884 * Allow user to specify configuration file name (-s option) 885 * 886 * @param[out] fn set to the name of the configuration file 887 */ 888 static struct GNUNET_GETOPT_CommandLineOption 889 option_cfgfile_src (char **fn) 890 { 891 struct GNUNET_GETOPT_CommandLineOption clo = { 892 .shortName = 's', 893 .name = "source-configuration", 894 .argumentHelp = "FILENAME", 895 .description = gettext_noop ( 896 "use configuration file FILENAME for the SOURCE database"), 897 .require_argument = 1, 898 .processor = &set_filename, 899 .scls = (void *) fn 900 }; 901 902 return clo; 903 } 904 905 906 /** 907 * Allow user to specify configuration file name (-d option) 908 * 909 * @param[out] fn set to the name of the configuration file 910 */ 911 static struct GNUNET_GETOPT_CommandLineOption 912 option_cfgfile_dst (char **fn) 913 { 914 struct GNUNET_GETOPT_CommandLineOption clo = { 915 .shortName = 'd', 916 .name = "destination-configuration", 917 .argumentHelp = "FILENAME", 918 .description = gettext_noop ( 919 "use configuration file FILENAME for the DESTINATION database"), 920 .require_argument = 1, 921 .processor = &set_filename, 922 .scls = (void *) fn 923 }; 924 925 return clo; 926 } 927 928 929 static struct GNUNET_CONFIGURATION_Handle * 930 load_config (const char *cfgfile) 931 { 932 struct GNUNET_CONFIGURATION_Handle *cfg; 933 934 cfg = GNUNET_CONFIGURATION_create (TALER_AUDITOR_project_data ()); 935 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 936 "Loading config file: %s\n", 937 cfgfile); 938 if (GNUNET_SYSERR == 939 GNUNET_CONFIGURATION_load (cfg, 940 cfgfile)) 941 { 942 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 943 "Malformed configuration file `%s', exit ...\n", 944 cfgfile); 945 GNUNET_CONFIGURATION_destroy (cfg); 946 return NULL; 947 } 948 return cfg; 949 } 950 951 952 /** 953 * Shutdown task. 954 * 955 * @param cls NULL, unused 956 */ 957 static void 958 do_shutdown (void *cls) 959 { 960 (void) cls; 961 if (NULL != sync_task) 962 { 963 GNUNET_SCHEDULER_cancel (sync_task); 964 sync_task = NULL; 965 } 966 } 967 968 969 /** 970 * Initial task. 971 * 972 * @param cls NULL, unused 973 */ 974 static void 975 run (void *cls) 976 { 977 (void) cls; 978 979 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 980 NULL); 981 sync_task = GNUNET_SCHEDULER_add_now (&do_sync, 982 NULL); 983 } 984 985 986 /** 987 * Setup plugins in #src and #dst and #run() the main 988 * logic with those plugins. 989 */ 990 static void 991 setup (struct GNUNET_CONFIGURATION_Handle *src_cfg, 992 struct GNUNET_CONFIGURATION_Handle *dst_cfg) 993 { 994 src = TALER_EXCHANGEDB_connect (src_cfg); 995 if (NULL == src) 996 { 997 global_ret = EXIT_NOTINSTALLED; 998 return; 999 } 1000 dst = TALER_EXCHANGEDB_connect (dst_cfg); 1001 if (NULL == dst) 1002 { 1003 global_ret = EXIT_NOTINSTALLED; 1004 TALER_EXCHANGEDB_disconnect (src); 1005 src = NULL; 1006 return; 1007 } 1008 GNUNET_SCHEDULER_run (&run, 1009 NULL); 1010 TALER_EXCHANGEDB_disconnect (src); 1011 src = NULL; 1012 TALER_EXCHANGEDB_disconnect (dst); 1013 dst = NULL; 1014 } 1015 1016 1017 /** 1018 * The main function of the taler-auditor-exchange tool. This tool is used 1019 * to add (or remove) an exchange's master key and base URL to the auditor's 1020 * database. 1021 * 1022 * @param argc number of arguments from the command line 1023 * @param argv command line arguments 1024 * @return 0 ok, non-zero on error 1025 */ 1026 int 1027 main (int argc, 1028 char *const *argv) 1029 { 1030 char *src_cfgfile = NULL; 1031 char *dst_cfgfile = NULL; 1032 char *level = GNUNET_strdup ("WARNING"); 1033 struct GNUNET_CONFIGURATION_Handle *src_cfg; 1034 struct GNUNET_CONFIGURATION_Handle *dst_cfg; 1035 const struct GNUNET_GETOPT_CommandLineOption options[] = { 1036 GNUNET_GETOPT_option_mandatory ( 1037 option_cfgfile_src (&src_cfgfile)), 1038 GNUNET_GETOPT_option_mandatory ( 1039 option_cfgfile_dst (&dst_cfgfile)), 1040 GNUNET_GETOPT_option_help ( 1041 TALER_AUDITOR_project_data (), 1042 gettext_noop ("Make a safe copy of an exchange database")), 1043 GNUNET_GETOPT_option_uint ( 1044 'b', 1045 "batch", 1046 "SIZE", 1047 gettext_noop ( 1048 "target SIZE for a the number of records to copy in one transaction"), 1049 &transaction_size), 1050 GNUNET_GETOPT_option_flag ( 1051 't', 1052 "terminate-when-synchronized", 1053 gettext_noop ( 1054 "terminate as soon as the databases are synchronized"), 1055 &exit_if_synced), 1056 GNUNET_GETOPT_option_version (VERSION), 1057 GNUNET_GETOPT_option_loglevel (&level), 1058 GNUNET_GETOPT_OPTION_END 1059 }; 1060 1061 TALER_gcrypt_init (); /* must trigger initialization manually at this point! */ 1062 { 1063 int ret; 1064 1065 ret = GNUNET_GETOPT_run ("taler-auditor-sync", 1066 options, 1067 argc, argv); 1068 if (GNUNET_NO == ret) 1069 return EXIT_SUCCESS; 1070 if (GNUNET_SYSERR == ret) 1071 return EXIT_INVALIDARGUMENT; 1072 } 1073 GNUNET_assert (GNUNET_OK == 1074 GNUNET_log_setup ("taler-auditor-sync", 1075 level, 1076 NULL)); 1077 GNUNET_free (level); 1078 /* suppress compiler warnings... */ 1079 GNUNET_assert (NULL != src_cfgfile); 1080 GNUNET_assert (NULL != dst_cfgfile); 1081 if (GNUNET_OK != 1082 check_tables ()) 1083 { 1084 GNUNET_break (0); 1085 GNUNET_free (src_cfgfile); 1086 GNUNET_free (dst_cfgfile); 1087 return EXIT_FAILURE; 1088 } 1089 if (0 == strcmp (src_cfgfile, 1090 dst_cfgfile)) 1091 { 1092 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 1093 "Source and destination configuration files must differ!\n"); 1094 return EXIT_INVALIDARGUMENT; 1095 } 1096 src_cfg = load_config (src_cfgfile); 1097 if (NULL == src_cfg) 1098 { 1099 GNUNET_free (src_cfgfile); 1100 GNUNET_free (dst_cfgfile); 1101 return EXIT_NOTCONFIGURED; 1102 } 1103 dst_cfg = load_config (dst_cfgfile); 1104 if (NULL == dst_cfg) 1105 { 1106 GNUNET_CONFIGURATION_destroy (src_cfg); 1107 GNUNET_free (src_cfgfile); 1108 GNUNET_free (dst_cfgfile); 1109 return EXIT_NOTCONFIGURED; 1110 } 1111 setup (src_cfg, 1112 dst_cfg); 1113 GNUNET_CONFIGURATION_destroy (src_cfg); 1114 GNUNET_CONFIGURATION_destroy (dst_cfg); 1115 GNUNET_free (src_cfgfile); 1116 GNUNET_free (dst_cfgfile); 1117 1118 return global_ret; 1119 } 1120 1121 1122 /* end of taler-auditor-sync.c */