taler-auditor-sync.c (17967B)
1 /* 2 This file is part of TALER 3 Copyright (C) 2020-2022 Taler Systems SA 4 5 TALER is free software; you can redistribute it and/or modify it under the 6 terms of the GNU General Public License as published by the Free Software 7 Foundation; either version 3, or (at your option) any later version. 8 9 TALER is distributed in the hope that it will be useful, but WITHOUT ANY 10 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 11 A PARTICULAR PURPOSE. See the GNU General Public License for more details. 12 13 You should have received a copy of the GNU General Public License along with 14 TALER; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 15 */ 16 /** 17 * @file taler-auditor-sync.c 18 * @brief Tool used by the auditor to make a 'safe' copy of the exchanges' database. 19 * @author Christian Grothoff 20 */ 21 #include "taler/platform.h" 22 #include "taler/taler_exchangedb_lib.h" 23 24 25 /** 26 * Handle to access the exchange's source database. 27 */ 28 static struct TALER_EXCHANGEDB_Plugin *src; 29 30 /** 31 * Handle to access the exchange's destination database. 32 */ 33 static struct TALER_EXCHANGEDB_Plugin *dst; 34 35 /** 36 * Return value from #main(). 37 */ 38 static int global_ret; 39 40 /** 41 * Main task to do synchronization. 42 */ 43 static struct GNUNET_SCHEDULER_Task *sync_task; 44 45 /** 46 * What is our target transaction size (number of records)? 47 */ 48 static unsigned int transaction_size = 512; 49 50 /** 51 * Number of records copied in this transaction. 52 */ 53 static unsigned long long actual_size; 54 55 /** 56 * Terminate once synchronization is achieved. 57 */ 58 static int exit_if_synced; 59 60 61 /** 62 * Information we track per replicated table. 63 */ 64 struct Table 65 { 66 /** 67 * Which table is this record about? 68 */ 69 enum TALER_EXCHANGEDB_ReplicatedTable rt; 70 71 /** 72 * Up to which record is the destination table synchronized. 73 */ 74 uint64_t start_serial; 75 76 /** 77 * Highest serial in the source table. 78 */ 79 uint64_t end_serial; 80 81 /** 82 * Marker for the end of the list of #tables. 83 */ 84 bool end; 85 }; 86 87 88 /** 89 * Information about replicated tables. 90 */ 91 static struct Table tables[] = { 92 { .rt = TALER_EXCHANGEDB_RT_DENOMINATIONS}, 93 { .rt = TALER_EXCHANGEDB_RT_DENOMINATION_REVOCATIONS}, 94 { .rt = TALER_EXCHANGEDB_RT_KYC_TARGETS}, 95 { .rt = TALER_EXCHANGEDB_RT_WIRE_TARGETS}, 96 { .rt = TALER_EXCHANGEDB_RT_LEGITIMIZATION_MEASURES}, 97 { .rt = TALER_EXCHANGEDB_RT_LEGITIMIZATION_OUTCOMES}, 98 { .rt = TALER_EXCHANGEDB_RT_LEGITIMIZATION_PROCESSES}, 99 { .rt = TALER_EXCHANGEDB_RT_RESERVES}, 100 { .rt = TALER_EXCHANGEDB_RT_RESERVES_IN}, 101 { .rt = TALER_EXCHANGEDB_RT_RESERVES_CLOSE}, 102 { .rt = TALER_EXCHANGEDB_RT_RESERVES_OPEN_REQUESTS}, 103 { .rt = TALER_EXCHANGEDB_RT_RESERVES_OPEN_DEPOSITS}, 104 { .rt = TALER_EXCHANGEDB_RT_WITHDRAW}, 105 { .rt = TALER_EXCHANGEDB_RT_AUDITORS}, 106 { .rt = TALER_EXCHANGEDB_RT_AUDITOR_DENOM_SIGS}, 107 { .rt = TALER_EXCHANGEDB_RT_EXCHANGE_SIGN_KEYS}, 108 { .rt = TALER_EXCHANGEDB_RT_SIGNKEY_REVOCATIONS}, 109 { .rt = TALER_EXCHANGEDB_RT_KNOWN_COINS}, 110 { .rt = TALER_EXCHANGEDB_RT_REFRESH}, 111 { .rt = TALER_EXCHANGEDB_RT_BATCH_DEPOSITS}, 112 { .rt = TALER_EXCHANGEDB_RT_COIN_DEPOSITS}, 113 { .rt = TALER_EXCHANGEDB_RT_REFUNDS}, 114 { .rt = TALER_EXCHANGEDB_RT_WIRE_OUT}, 115 { .rt = TALER_EXCHANGEDB_RT_AGGREGATION_TRACKING}, 116 { .rt = TALER_EXCHANGEDB_RT_WIRE_FEE}, 117 { .rt = TALER_EXCHANGEDB_RT_GLOBAL_FEE}, 118 { .rt = TALER_EXCHANGEDB_RT_RECOUP}, 119 { .rt = TALER_EXCHANGEDB_RT_RECOUP_REFRESH }, 120 { .rt = TALER_EXCHANGEDB_RT_EXTENSIONS}, 121 { .rt = TALER_EXCHANGEDB_RT_POLICY_DETAILS }, 122 { .rt = TALER_EXCHANGEDB_RT_POLICY_FULFILLMENTS }, 123 { .rt = TALER_EXCHANGEDB_RT_PURSE_REQUESTS}, 124 { .rt = TALER_EXCHANGEDB_RT_PURSE_DECISION}, 125 { .rt = TALER_EXCHANGEDB_RT_PURSE_MERGES}, 126 { .rt = TALER_EXCHANGEDB_RT_PURSE_DEPOSITS}, 127 { .rt = TALER_EXCHANGEDB_RT_ACCOUNT_MERGES}, 128 { .rt = TALER_EXCHANGEDB_RT_HISTORY_REQUESTS}, 129 { .rt = TALER_EXCHANGEDB_RT_CLOSE_REQUESTS}, 130 { .rt = TALER_EXCHANGEDB_RT_WADS_OUT}, 131 { .rt = TALER_EXCHANGEDB_RT_WADS_OUT_ENTRIES}, 132 { .rt = TALER_EXCHANGEDB_RT_WADS_IN}, 133 { .rt = TALER_EXCHANGEDB_RT_WADS_IN_ENTRIES}, 134 { .rt = TALER_EXCHANGEDB_RT_PROFIT_DRAINS}, 135 { .end = true } 136 }; 137 138 139 /** 140 * Closure for #do_insert. 141 */ 142 struct InsertContext 143 { 144 /** 145 * Table we are replicating. 146 */ 147 struct Table *table; 148 149 /** 150 * Set to error if insertion created an error. 151 */ 152 enum GNUNET_DB_QueryStatus qs; 153 }; 154 155 156 /** 157 * Function called on data to replicate in the auditor's database. 158 * 159 * @param cls closure, a `struct InsertContext` 160 * @param td record from an exchange table 161 * @return #GNUNET_OK to continue to iterate, 162 * #GNUNET_SYSERR to fail with an error 163 */ 164 static enum GNUNET_GenericReturnValue 165 do_insert (void *cls, 166 const struct TALER_EXCHANGEDB_TableData *td) 167 { 168 struct InsertContext *ctx = cls; 169 enum GNUNET_DB_QueryStatus qs; 170 171 if (0 >= ctx->qs) 172 return GNUNET_SYSERR; 173 qs = dst->insert_records_by_table (dst->cls, 174 td); 175 if (0 >= qs) 176 { 177 switch (qs) 178 { 179 case GNUNET_DB_STATUS_SUCCESS_ONE_RESULT: 180 GNUNET_assert (0); 181 break; 182 case GNUNET_DB_STATUS_SUCCESS_NO_RESULTS: 183 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 184 "Failed to insert record into table %d: no change\n", 185 td->table); 186 break; 187 case GNUNET_DB_STATUS_SOFT_ERROR: 188 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 189 "Serialization error inserting record into table %d (will retry)\n", 190 td->table); 191 break; 192 case GNUNET_DB_STATUS_HARD_ERROR: 193 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 194 "Failed to insert record into table %d: hard error\n", 195 td->table); 196 break; 197 } 198 ctx->qs = qs; 199 return GNUNET_SYSERR; 200 } 201 actual_size++; 202 ctx->table->start_serial = td->serial; 203 return GNUNET_OK; 204 } 205 206 207 /** 208 * Run one replication transaction. 209 * 210 * @return #GNUNET_OK on success, #GNUNET_SYSERR to rollback 211 */ 212 static enum GNUNET_GenericReturnValue 213 transact (void) 214 { 215 struct InsertContext ctx = { 216 .qs = GNUNET_DB_STATUS_SUCCESS_ONE_RESULT 217 }; 218 219 if (0 > 220 src->start (src->cls, 221 "lookup src serials")) 222 return GNUNET_SYSERR; 223 for (unsigned int i = 0; ! tables[i].end; i++) 224 src->lookup_serial_by_table (src->cls, 225 tables[i].rt, 226 &tables[i].end_serial); 227 src->rollback (src->cls); 228 if (GNUNET_OK != 229 dst->start (dst->cls, 230 "lookup dst serials")) 231 return GNUNET_SYSERR; 232 for (unsigned int i = 0; ! tables[i].end; i++) 233 dst->lookup_serial_by_table (dst->cls, 234 tables[i].rt, 235 &tables[i].start_serial); 236 dst->rollback (dst->cls); 237 for (unsigned int i = 0; ! tables[i].end; i++) 238 { 239 struct Table *table = &tables[i]; 240 241 if (table->start_serial == table->end_serial) 242 continue; 243 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 244 "Replicating table %d from %llu to %llu\n", 245 i, 246 (unsigned long long) table->start_serial, 247 (unsigned long long) table->end_serial); 248 ctx.table = table; 249 while (table->start_serial < table->end_serial) 250 { 251 enum GNUNET_DB_QueryStatus qs; 252 253 if (GNUNET_OK != 254 src->start (src->cls, 255 "copy table (src)")) 256 return GNUNET_SYSERR; 257 if (GNUNET_OK != 258 dst->start (dst->cls, 259 "copy table (dst)")) 260 return GNUNET_SYSERR; 261 qs = src->lookup_records_by_table (src->cls, 262 table->rt, 263 table->start_serial, 264 &do_insert, 265 &ctx); 266 if (ctx.qs < 0) 267 qs = ctx.qs; 268 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 269 { 270 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 271 "Failed to lookup records from table %d: hard error\n", 272 i); 273 global_ret = EXIT_FAILURE; 274 return GNUNET_SYSERR; 275 } 276 if (GNUNET_DB_STATUS_SOFT_ERROR == qs) 277 { 278 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 279 "Serialization error looking up records from table %d (will retry)\n", 280 i); 281 return GNUNET_SYSERR; /* will retry */ 282 } 283 if (0 == qs) 284 { 285 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 286 "Failed to lookup records from table %d: no results\n", 287 i); 288 GNUNET_break (0); /* should be impossible */ 289 global_ret = EXIT_FAILURE; 290 return GNUNET_SYSERR; 291 } 292 if (0 == ctx.qs) 293 return GNUNET_SYSERR; /* insertion failed, maybe record existed? try again */ 294 src->rollback (src->cls); 295 qs = dst->commit (dst->cls); 296 if (GNUNET_DB_STATUS_SOFT_ERROR == qs) 297 { 298 GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 299 "Serialization error committing transaction on table %d (will retry)\n", 300 i); 301 continue; 302 } 303 if (GNUNET_DB_STATUS_HARD_ERROR == qs) 304 { 305 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 306 "Hard error committing transaction on table %d\n", 307 i); 308 global_ret = EXIT_FAILURE; 309 return GNUNET_SYSERR; 310 } 311 } 312 } 313 /* we do not care about conflicting UPDATEs to src table, so safe to just rollback */ 314 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 315 "Sync pass completed successfully with %llu updates\n", 316 actual_size); 317 return GNUNET_OK; 318 } 319 320 321 /** 322 * Task to do the actual synchronization work. 323 * 324 * @param cls NULL, unused 325 */ 326 static void 327 do_sync (void *cls) 328 { 329 static struct GNUNET_TIME_Relative delay; 330 331 (void) cls; 332 sync_task = NULL; 333 actual_size = 0; 334 if (GNUNET_SYSERR == 335 src->preflight (src->cls)) 336 { 337 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 338 "Failed to begin transaction with data source. Exiting\n"); 339 return; 340 } 341 if (GNUNET_SYSERR == 342 dst->preflight (dst->cls)) 343 { 344 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 345 "Failed to begin transaction with data destination. Exiting\n"); 346 return; 347 } 348 if (GNUNET_OK != transact ()) 349 { 350 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 351 "Transaction failed, rolling back\n"); 352 src->rollback (src->cls); 353 dst->rollback (dst->cls); 354 } 355 if (0 != global_ret) 356 { 357 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 358 "Transaction failed permanently, exiting\n"); 359 return; 360 } 361 if ( (0 == actual_size) && 362 (exit_if_synced) ) 363 { 364 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 365 "Databases are synchronized. Exiting\n"); 366 return; 367 } 368 if (actual_size < transaction_size / 2) 369 { 370 delay = GNUNET_TIME_STD_BACKOFF (delay); 371 } 372 else if (actual_size >= transaction_size) 373 { 374 delay = GNUNET_TIME_UNIT_ZERO; 375 } 376 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 377 "Next sync pass in %s\n", 378 GNUNET_STRINGS_relative_time_to_string (delay, 379 GNUNET_YES)); 380 sync_task = GNUNET_SCHEDULER_add_delayed (delay, 381 &do_sync, 382 NULL); 383 } 384 385 386 /** 387 * Set an option of type 'char *' from the command line with 388 * filename expansion a la #GNUNET_STRINGS_filename_expand(). 389 * 390 * @param ctx command line processing context 391 * @param scls additional closure (will point to the `char *`, 392 * which will be allocated) 393 * @param option name of the option 394 * @param value actual value of the option (a string) 395 * @return #GNUNET_OK 396 */ 397 static enum GNUNET_GenericReturnValue 398 set_filename (struct GNUNET_GETOPT_CommandLineProcessorContext *ctx, 399 void *scls, 400 const char *option, 401 const char *value) 402 { 403 char **val = scls; 404 405 (void) ctx; 406 (void) option; 407 GNUNET_assert (NULL != value); 408 GNUNET_free (*val); 409 *val = GNUNET_STRINGS_filename_expand (value); 410 return GNUNET_OK; 411 } 412 413 414 /** 415 * Allow user to specify configuration file name (-s option) 416 * 417 * @param[out] fn set to the name of the configuration file 418 */ 419 static struct GNUNET_GETOPT_CommandLineOption 420 option_cfgfile_src (char **fn) 421 { 422 struct GNUNET_GETOPT_CommandLineOption clo = { 423 .shortName = 's', 424 .name = "source-configuration", 425 .argumentHelp = "FILENAME", 426 .description = gettext_noop ( 427 "use configuration file FILENAME for the SOURCE database"), 428 .require_argument = 1, 429 .processor = &set_filename, 430 .scls = (void *) fn 431 }; 432 433 return clo; 434 } 435 436 437 /** 438 * Allow user to specify configuration file name (-d option) 439 * 440 * @param[out] fn set to the name of the configuration file 441 */ 442 static struct GNUNET_GETOPT_CommandLineOption 443 option_cfgfile_dst (char **fn) 444 { 445 struct GNUNET_GETOPT_CommandLineOption clo = { 446 .shortName = 'd', 447 .name = "destination-configuration", 448 .argumentHelp = "FILENAME", 449 .description = gettext_noop ( 450 "use configuration file FILENAME for the DESTINATION database"), 451 .require_argument = 1, 452 .processor = &set_filename, 453 .scls = (void *) fn 454 }; 455 456 return clo; 457 } 458 459 460 static struct GNUNET_CONFIGURATION_Handle * 461 load_config (const char *cfgfile) 462 { 463 struct GNUNET_CONFIGURATION_Handle *cfg; 464 465 cfg = GNUNET_CONFIGURATION_create (TALER_AUDITOR_project_data ()); 466 GNUNET_log (GNUNET_ERROR_TYPE_INFO, 467 "Loading config file: %s\n", 468 cfgfile); 469 if (GNUNET_SYSERR == 470 GNUNET_CONFIGURATION_load (cfg, 471 cfgfile)) 472 { 473 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 474 "Malformed configuration file `%s', exit ...\n", 475 cfgfile); 476 GNUNET_CONFIGURATION_destroy (cfg); 477 return NULL; 478 } 479 return cfg; 480 } 481 482 483 /** 484 * Shutdown task. 485 * 486 * @param cls NULL, unused 487 */ 488 static void 489 do_shutdown (void *cls) 490 { 491 (void) cls; 492 if (NULL != sync_task) 493 { 494 GNUNET_SCHEDULER_cancel (sync_task); 495 sync_task = NULL; 496 } 497 } 498 499 500 /** 501 * Initial task. 502 * 503 * @param cls NULL, unused 504 */ 505 static void 506 run (void *cls) 507 { 508 (void) cls; 509 510 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, 511 NULL); 512 sync_task = GNUNET_SCHEDULER_add_now (&do_sync, 513 NULL); 514 } 515 516 517 /** 518 * Setup plugins in #src and #dst and #run() the main 519 * logic with those plugins. 520 */ 521 static void 522 setup (struct GNUNET_CONFIGURATION_Handle *src_cfg, 523 struct GNUNET_CONFIGURATION_Handle *dst_cfg) 524 { 525 src = TALER_EXCHANGEDB_plugin_load (src_cfg, 526 false); 527 if (NULL == src) 528 { 529 global_ret = EXIT_NOTINSTALLED; 530 return; 531 } 532 dst = TALER_EXCHANGEDB_plugin_load (dst_cfg, 533 false); 534 if (NULL == dst) 535 { 536 global_ret = EXIT_NOTINSTALLED; 537 TALER_EXCHANGEDB_plugin_unload (src); 538 src = NULL; 539 return; 540 } 541 GNUNET_SCHEDULER_run (&run, 542 NULL); 543 TALER_EXCHANGEDB_plugin_unload (src); 544 src = NULL; 545 TALER_EXCHANGEDB_plugin_unload (dst); 546 dst = NULL; 547 } 548 549 550 /** 551 * The main function of the taler-auditor-exchange tool. This tool is used 552 * to add (or remove) an exchange's master key and base URL to the auditor's 553 * database. 554 * 555 * @param argc number of arguments from the command line 556 * @param argv command line arguments 557 * @return 0 ok, non-zero on error 558 */ 559 int 560 main (int argc, 561 char *const *argv) 562 { 563 char *src_cfgfile = NULL; 564 char *dst_cfgfile = NULL; 565 char *level = GNUNET_strdup ("WARNING"); 566 struct GNUNET_CONFIGURATION_Handle *src_cfg; 567 struct GNUNET_CONFIGURATION_Handle *dst_cfg; 568 const struct GNUNET_GETOPT_CommandLineOption options[] = { 569 GNUNET_GETOPT_option_mandatory ( 570 option_cfgfile_src (&src_cfgfile)), 571 GNUNET_GETOPT_option_mandatory ( 572 option_cfgfile_dst (&dst_cfgfile)), 573 GNUNET_GETOPT_option_help ( 574 TALER_AUDITOR_project_data (), 575 gettext_noop ("Make a safe copy of an exchange database")), 576 GNUNET_GETOPT_option_uint ( 577 'b', 578 "batch", 579 "SIZE", 580 gettext_noop ( 581 "target SIZE for a the number of records to copy in one transaction"), 582 &transaction_size), 583 GNUNET_GETOPT_option_flag ( 584 't', 585 "terminate-when-synchronized", 586 gettext_noop ( 587 "terminate as soon as the databases are synchronized"), 588 &exit_if_synced), 589 GNUNET_GETOPT_option_version (VERSION "-" VCS_VERSION), 590 GNUNET_GETOPT_option_loglevel (&level), 591 GNUNET_GETOPT_OPTION_END 592 }; 593 594 TALER_gcrypt_init (); /* must trigger initialization manually at this point! */ 595 { 596 int ret; 597 598 ret = GNUNET_GETOPT_run ("taler-auditor-sync", 599 options, 600 argc, argv); 601 if (GNUNET_NO == ret) 602 return EXIT_SUCCESS; 603 if (GNUNET_SYSERR == ret) 604 return EXIT_INVALIDARGUMENT; 605 } 606 GNUNET_assert (GNUNET_OK == 607 GNUNET_log_setup ("taler-auditor-sync", 608 level, 609 NULL)); 610 GNUNET_free (level); 611 /* suppress compiler warnings... */ 612 GNUNET_assert (NULL != src_cfgfile); 613 GNUNET_assert (NULL != dst_cfgfile); 614 if (0 == strcmp (src_cfgfile, 615 dst_cfgfile)) 616 { 617 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, 618 "Source and destination configuration files must differ!\n"); 619 return EXIT_INVALIDARGUMENT; 620 } 621 src_cfg = load_config (src_cfgfile); 622 if (NULL == src_cfg) 623 { 624 GNUNET_free (src_cfgfile); 625 GNUNET_free (dst_cfgfile); 626 return EXIT_NOTCONFIGURED; 627 } 628 dst_cfg = load_config (dst_cfgfile); 629 if (NULL == dst_cfg) 630 { 631 GNUNET_CONFIGURATION_destroy (src_cfg); 632 GNUNET_free (src_cfgfile); 633 GNUNET_free (dst_cfgfile); 634 return EXIT_NOTCONFIGURED; 635 } 636 setup (src_cfg, 637 dst_cfg); 638 GNUNET_CONFIGURATION_destroy (src_cfg); 639 GNUNET_CONFIGURATION_destroy (dst_cfg); 640 GNUNET_free (src_cfgfile); 641 GNUNET_free (dst_cfgfile); 642 643 return global_ret; 644 } 645 646 647 /* end of taler-auditor-sync.c */