test-kyc.sh (64558B)
1 #!/bin/bash 2 # 3 # This file is part of TALER 4 # Copyright (C) 2014-2026 Taler Systems SA 5 # 6 # TALER is free software; you can redistribute it and/or modify it under the 7 # terms of the GNU General Public License as published by the Free Software 8 # Foundation; either version 3, or (at your option) any later version. 9 # 10 # TALER is distributed in the hope that it will be useful, but WITHOUT ANY 11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 12 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 13 # 14 # You should have received a copy of the GNU General Public License along with 15 # TALER; see the file COPYING. If not, If not, see <http://www.gnu.org/license> 16 # 17 # 18 # shellcheck disable=SC2317 19 # shellcheck disable=SC1091 20 # 21 # 22 # Setup database which was generated from a perfectly normal 23 # exchange-wallet-merchant interaction on a KYC-enabled exchange and run 24 # the auditor against it. 25 # 26 # The reference database (see generate-kyc-basedb.sh) contains two merchant 27 # instances that were paid the same amount and both proved that they own 28 # their bank account with a KYCAUTH wire transfer, but only one of them 29 # then passed the KYC form. The exchange therefore wired the funds of the 30 # first one out and is still holding the funds of the second one. 31 # 32 # The auditor has almost no KYC-specific logic of its own -- see issues.txt 33 # for what it could check and does not -- so this test has two jobs: 34 # 35 # 1. the KYC artefacts (KYCAUTH credits, legitimization measures and 36 # outcomes, an aggregation the exchange lawfully withheld) must not 37 # make the auditor invent findings, and the KYCAUTH totals it does 38 # track -- total_kycauth_in from the bank, total_kycauth_revenue from 39 # the exchange -- must be exactly right; 40 # 2. fault detection must still work on a KYC-enabled exchange, so the 41 # later tests inject the same kinds of faults test-auditor.sh does and 42 # require the auditor to flag them. Tests 2, 8 and 9 pull the two 43 # KYCAUTH totals apart in each of the ways they can disagree, which is 44 # what the reconciliation between them exists to catch. 45 # 46 # Requires 'jq' tool and Postgres superuser rights! 47 set -eu 48 #set -x 49 50 # Set of numbers for all the testcases. 51 # When adding new tests, increase the last number: 52 ALL_TESTS=$(seq 0 16) 53 54 # $TESTS determines which tests we should run. 55 # This construction is used to make it easy to 56 # only run a subset of the tests. To only run a subset, 57 # pass the numbers of the tests to run as the FIRST 58 # argument to test-kyc.sh, i.e.: 59 # 60 # $ test-kyc.sh "1 3" 61 # 62 # to run tests 1 and 3 only. By default, all tests are run. 63 # 64 TESTS=${1:-$ALL_TESTS} 65 66 export TALER_AUDITOR_TOKEN="secret-token:D4CST1Z6AHN3RT03M0T9NSTF2QGHTB5ZD2D3RYZB4HAWG8SX0JEFWBXCKXZHMB7Y3Z7KVFW0B3XPXD5BHCFP8EB0R6CNH2KAWDWVET0" 67 export TALER_AUDITOR_SALT="64S36D1N6RVKGC9J6CT3ADHQ70RK4CSM6MV3EE1H68SK8D9P6WW32CHK6GTKCDSR64S36D1N6RVKGC9J6CT3ADHQ70RK4CSM6MV3EE0" 68 69 # Global variable to run the auditor processes under valgrind 70 # VALGRIND=valgrind 71 VALGRIND="" 72 73 . setup.sh 74 75 76 # Cleanup exchange and libeufin between runs. 77 function cleanup() 78 { 79 if [ -n "${EPID:-}" ] 80 then 81 echo -n "Stopping exchange $EPID..." 82 kill -TERM "$EPID" 83 wait "$EPID" || true 84 echo "DONE" 85 unset EPID 86 fi 87 stop_libeufin &> /dev/null 88 } 89 90 # Cleanup to run whenever we exit 91 function exit_cleanup() 92 { 93 jobs 94 if [ -n "${POSTGRES_PATH:-}" ] 95 then 96 echo -n "Stopping Postgres at ${POSTGRES_PATH} ..." 97 "${POSTGRES_PATH}/pg_ctl" \ 98 -D "$TMPDIR" \ 99 --log="${MY_TMP_DIR}/pg_ctl.log" \ 100 stop \ 101 &> ${MY_TMP_DIR}/pg_ctl.out \ 102 || true 103 echo "DONE" 104 fi 105 echo -n "Running exit-cleanup ..." 106 cleanup 107 for n in $(jobs -p) 108 do 109 kill "$n" 2> /dev/null || true 110 done 111 wait || true 112 echo "DONE" 113 } 114 115 # Install cleanup handler (except for kill -9) 116 trap exit_cleanup EXIT 117 118 119 function await_bank () { 120 for n in $(seq 1 80) 121 do 122 echo -n "." 123 sleep 0.1 124 OK=1 125 wget http://localhost:8082/ \ 126 -o /dev/null \ 127 -O /dev/null \ 128 >/dev/null \ 129 && break 130 OK=0 131 done 132 if [ 1 != "$OK" ] 133 then 134 exit_skip "Failed to launch libeufin-bank" 135 fi 136 } 137 138 # Operations to run before the actual audit 139 function pre_audit () { 140 # Launch bank 141 echo -n "Launching libeufin-bank" 142 export CONF 143 export MY_TMP_DIR 144 launch_libeufin 145 await_bank 146 echo " DONE" 147 148 if [ "${1:-no}" = "aggregator" ] 149 then 150 # NO '-y' here: that is --kyc-off, which makes the aggregator 151 # wire funds out without consulting KYC at all and would defeat 152 # the point of this test. 153 echo -n "Running exchange aggregator ..." 154 taler-exchange-aggregator \ 155 -L "INFO" \ 156 -t \ 157 -c "$CONF" \ 158 2> "${MY_TMP_DIR}/aggregator.log" \ 159 || exit_fail "FAIL" 160 echo " DONE" 161 echo -n "Running exchange closer ..." 162 taler-exchange-closer \ 163 -L "INFO" \ 164 -t \ 165 -c "$CONF" \ 166 2> "${MY_TMP_DIR}/closer.log" \ 167 || exit_fail "FAIL" 168 echo " DONE" 169 echo -n "Running exchange transfer ..." 170 taler-exchange-transfer \ 171 -L "INFO" \ 172 -t \ 173 -c "$CONF" \ 174 2> "${MY_TMP_DIR}/transfer.log" \ 175 || exit_fail "FAIL" 176 echo " DONE" 177 fi 178 } 179 180 # actual audit run 181 function audit_only () { 182 # Run the auditor! 183 echo -n "Running audit(s) ..." 184 185 # Restart so that first run is always fresh, and second one is incremental 186 taler-auditor-dbinit \ 187 -r \ 188 -c "$CONF" 189 $VALGRIND taler-helper-auditor-aml \ 190 -i \ 191 -L DEBUG \ 192 -c "$CONF" \ 193 -t \ 194 > "${MY_TMP_DIR}/test-audit-aml.out" \ 195 2> "${MY_TMP_DIR}/test-audit-aml.err" \ 196 || exit_fail "aml audit failed (see ${MY_TMP_DIR}/test-audit-aml.*)" 197 echo -n "." 198 $VALGRIND taler-helper-auditor-aml \ 199 -i \ 200 -L DEBUG \ 201 -c "$CONF" \ 202 -t \ 203 > "${MY_TMP_DIR}/test-audit-aml-inc.out" \ 204 2> "${MY_TMP_DIR}/test-audit-aml-inc.err" \ 205 || exit_fail "incremental aml audit failed (see ${MY_TMP_DIR}/test-audit-aml-inc.*)" 206 echo -n "." 207 $VALGRIND taler-helper-auditor-aggregation \ 208 -L DEBUG \ 209 -c "$CONF" \ 210 -t \ 211 > "${MY_TMP_DIR}/test-audit-aggregation.out" \ 212 2> "${MY_TMP_DIR}/test-audit-aggregation.err" \ 213 || exit_fail "aggregation audit failed (see ${MY_TMP_DIR}/test-audit-aggregation.*)" 214 echo -n "." 215 $VALGRIND taler-helper-auditor-aggregation \ 216 -L DEBUG \ 217 -c "$CONF" \ 218 -t \ 219 > "${MY_TMP_DIR}/test-audit-aggregation-inc.out" \ 220 2> "${MY_TMP_DIR}/test-audit-aggregation-inc.err" \ 221 || exit_fail "incremental aggregation audit failed (see ${MY_TMP_DIR}/test-audit-aggregation-inc.*)" 222 echo -n "." 223 $VALGRIND taler-helper-auditor-coins \ 224 -L DEBUG \ 225 -c "$CONF" \ 226 -t \ 227 > "${MY_TMP_DIR}/test-audit-coins.out" \ 228 2> "${MY_TMP_DIR}/test-audit-coins.err" \ 229 || exit_fail "coin audit failed (see ${MY_TMP_DIR}/test-audit-coins.*)" 230 echo -n "." 231 $VALGRIND taler-helper-auditor-coins \ 232 -L DEBUG \ 233 -c "$CONF" \ 234 -t \ 235 > "${MY_TMP_DIR}/test-audit-coins-inc.out" \ 236 2> "${MY_TMP_DIR}/test-audit-coins-inc.err" \ 237 || exit_fail "incremental coin audit failed (see ${MY_TMP_DIR}/test-audit-coins-inc.*)" 238 echo -n "." 239 $VALGRIND taler-helper-auditor-deposits \ 240 -L DEBUG \ 241 -c "$CONF" \ 242 -t \ 243 > "${MY_TMP_DIR}/test-audit-deposits.out" \ 244 2> "${MY_TMP_DIR}/test-audit-deposits.err" \ 245 || exit_fail "deposits audit failed (see ${MY_TMP_DIR}/test-audit-deposits.*)" 246 echo -n "." 247 $VALGRIND taler-helper-auditor-deposits \ 248 -L DEBUG \ 249 -c "$CONF" \ 250 -t \ 251 > "${MY_TMP_DIR}/test-audit-deposits-inc.out" \ 252 2> "${MY_TMP_DIR}/test-audit-deposits-inc.err" \ 253 || exit_fail "incremental deposits audit failed (see ${MY_TMP_DIR}/test-audit-deposits-inc.*)" 254 echo -n "." 255 $VALGRIND taler-helper-auditor-reserves \ 256 -i \ 257 -L DEBUG \ 258 -c "$CONF" \ 259 -t \ 260 > "${MY_TMP_DIR}/test-audit-reserves.out" \ 261 2> "${MY_TMP_DIR}/test-audit-reserves.err" \ 262 || exit_fail "reserves audit failed (see ${MY_TMP_DIR}/test-audit-reserves.*)" 263 echo -n "." 264 $VALGRIND taler-helper-auditor-reserves \ 265 -i \ 266 -L DEBUG \ 267 -c "$CONF" \ 268 -t \ 269 > "${MY_TMP_DIR}/test-audit-reserves-inc.out" \ 270 2> "${MY_TMP_DIR}/test-audit-reserves-inc.err" \ 271 || exit_fail "incremental reserves audit failed (see ${MY_TMP_DIR}/test-audit-reserves-inc.*)" 272 echo -n "." 273 $VALGRIND taler-helper-auditor-wire-credit \ 274 -i \ 275 -L DEBUG \ 276 -c "$CONF" \ 277 -t \ 278 > "${MY_TMP_DIR}/test-audit-wire-credit.out" \ 279 2> "${MY_TMP_DIR}/test-audit-wire-credit.err" \ 280 || exit_fail "wire credit audit failed (see ${MY_TMP_DIR}/test-audit-wire-credit.*)" 281 echo -n "." 282 $VALGRIND taler-helper-auditor-wire-credit \ 283 -i \ 284 -L DEBUG \ 285 -c "$CONF" \ 286 -t \ 287 > "${MY_TMP_DIR}/test-audit-wire-credit-inc.out" \ 288 2> "${MY_TMP_DIR}/test-audit-wire-credit-inc.err" \ 289 || exit_fail "wire credit audit inc failed (see ${MY_TMP_DIR}/test-audit-wire-credit-inc.*)" 290 echo -n "." 291 $VALGRIND taler-helper-auditor-wire-debit \ 292 -i \ 293 -L DEBUG \ 294 -c "$CONF" \ 295 -t \ 296 > "${MY_TMP_DIR}/test-audit-wire-debit.out" \ 297 2> "${MY_TMP_DIR}/test-audit-wire-debit.err" \ 298 || exit_fail "wire debit audit failed (see ${MY_TMP_DIR}/test-audit-wire-debit.*)" 299 echo -n "." 300 $VALGRIND taler-helper-auditor-wire-debit \ 301 -i \ 302 -L DEBUG \ 303 -c "$CONF" \ 304 -t \ 305 > "${MY_TMP_DIR}/test-audit-wire-debit-inc.out" \ 306 2> "${MY_TMP_DIR}/test-audit-wire-debit-inc.err" \ 307 || exit_fail "wire debit audit inc failed (see ${MY_TMP_DIR}/test-audit-wire-debit-inc.*)" 308 echo -n "." 309 $VALGRIND taler-helper-auditor-purses \ 310 -i \ 311 -L DEBUG \ 312 -c "$CONF" \ 313 -t \ 314 > "${MY_TMP_DIR}/test-audit-purses.out" \ 315 2> "${MY_TMP_DIR}/test-audit-purses.err" \ 316 || exit_fail "audit purses failed" 317 echo -n "." 318 $VALGRIND taler-helper-auditor-purses \ 319 -i \ 320 -L DEBUG \ 321 -c "$CONF" \ 322 -t \ 323 > "${MY_TMP_DIR}/test-audit-purses-inc.out" \ 324 2> "${MY_TMP_DIR}/test-audit-purses-inc.err" \ 325 || exit_fail "audit purses inc failed" 326 echo -n "." 327 $VALGRIND taler-helper-auditor-transfer \ 328 -i \ 329 -L DEBUG \ 330 -c "$CONF" \ 331 -t \ 332 > "${MY_TMP_DIR}/test-audit-transfer.out" \ 333 2> "${MY_TMP_DIR}/test-audit-transfer.err" \ 334 || exit_fail "audit transfer failed" 335 echo -n "." 336 $VALGRIND taler-helper-auditor-transfer \ 337 -i \ 338 -L DEBUG \ 339 -c "$CONF" \ 340 -t \ 341 > "${MY_TMP_DIR}/test-audit-transfer-inc.out" \ 342 2> "${MY_TMP_DIR}/test-audit-transfer-inc.err" \ 343 || exit_fail "audit transfer inc failed" 344 echo -n "." 345 346 echo " DONE" 347 } 348 349 350 # Cleanup to run after the auditor 351 function post_audit () { 352 taler-exchange-dbinit \ 353 -c "$CONF" \ 354 -g \ 355 || exit_fail "exchange DB GC failed" 356 cleanup 357 } 358 359 360 # Run audit process on current database, including report 361 # generation. Pass "aggregator" as $1 to run 362 # $ taler-exchange-aggregator 363 # before auditor (to trigger pending wire transfers). 364 # Pass "drain" as $2 to run a drain operation as well. 365 function run_audit () { 366 pre_audit "${1:-no}" 367 audit_only 368 post_audit 369 } 370 371 372 function stop_auditor_httpd() { 373 if [ -n "${APID:-}" ] 374 then 375 echo -n "Stopping auditor $APID..." 376 kill -TERM "$APID" 377 wait "$APID" || true 378 echo "DONE" 379 unset APID 380 fi 381 } 382 383 384 # Do a full reload of the (original) database 385 function full_reload() 386 { 387 echo -n "Doing full reload of the database (loading ${BASEDB}.sql into $DB at ${PGHOST:-})... " 388 dropdb -f "$DB" &>> ${MY_TMP_DIR}/drop.log || true 389 createdb -T template0 "$DB" \ 390 || exit_skip "could not create database $DB (at ${PGHOST:-})" 391 # Import pre-generated database, -q(ietly) using single (-1) transaction 392 psql -Aqt "$DB" \ 393 -q \ 394 -1 \ 395 -f "${BASEDB}.sql" \ 396 &>> ${MY_TMP_DIR}/postgresql-reload.log \ 397 || exit_skip "Failed to load database $DB from ${BASEDB}.sql" 398 echo "DONE" 399 # Technically, this call shouldn't be needed as libeufin should already be stopped here... 400 stop_libeufin 401 stop_auditor_httpd 402 } 403 404 function run_auditor_httpd() { 405 echo -n "Starting auditor..." 406 $VALGRIND taler-auditor-httpd \ 407 -c "${CONF}" \ 408 -L INFO \ 409 2> "${MY_TMP_DIR}/auditor-httpd.err" & 410 APID=$! 411 412 # Wait for auditor service to be available 413 for n in $(seq 1 50) 414 do 415 echo -n "." 416 sleep 0.2 417 OK=0 418 # auditor 419 wget "http://localhost:8083/config" \ 420 -o /dev/null \ 421 -O /dev/null \ 422 >/dev/null \ 423 || continue 424 OK=1 425 break 426 done 427 echo "... DONE." 428 } 429 430 431 function check_auditor_running() { 432 ARUNSTATUS=$(curl -Is http://localhost:8083/config | head -1) 433 if [ -n "${ARUNSTATUS:-}" ] 434 then 435 echo "Auditor running" 436 else 437 echo "Auditor not running, starting it" 438 run_auditor_httpd 439 fi 440 unset ARUNSTATUS 441 } 442 443 function call_endpoint() { 444 if [ -n "${2+x}" ] 445 then 446 curl -s -H "Accept: application/json" -H "Authorization: Bearer ${TALER_AUDITOR_TOKEN}" -o "${MY_TMP_DIR}/${2}.json" "localhost:8083/monitoring/${1}?limit=50&balance_key=${2}" 447 echo "endpoint ${1} called (with balance_key)... " 448 else 449 curl -s -H "Accept: application/json" -H "Authorization: Bearer ${TALER_AUDITOR_TOKEN}" -o "${MY_TMP_DIR}/${1}.json" "localhost:8083/monitoring/${1}?limit=50" 450 echo "endpoint ${1} called... " 451 fi 452 } 453 454 455 function check_balance() { 456 call_endpoint "balances" "$1" 457 BAL=$(jq -r .balances[0].balance_value < "${MY_TMP_DIR}/${1}.json") 458 if [ "$BAL" != "$2" ] 459 then 460 exit_fail "$3 (got $BAL, wanted $2)" 461 fi 462 echo "PASS" 463 } 464 465 466 function check_not_balance() { 467 call_endpoint "balances" "$1" 468 BAL=$(jq -r .balances[0].balance_value < "${MY_TMP_DIR}/${1}.json") 469 if [ "$BAL" = "$2" ] 470 then 471 exit_fail "$3 (got $BAL, wanted NOT $2)" 472 fi 473 echo "PASS" 474 } 475 476 477 function check_report() { 478 call_endpoint "$1" 479 NAME=$(echo "$1" | tr '-' '_') 480 # shellcheck disable=SC2086 481 VAL=$(jq -r .\"${NAME}\"[0].\"$2\" < "${MY_TMP_DIR}/${1}.json") 482 if [ "$VAL" != "$3" ] 483 then 484 exit_fail "$1::$2 (got $VAL, wanted $3)" 485 fi 486 echo "PASS" 487 } 488 489 # Check that at least one entry of report $1 has field $2 set to $3. 490 # Unlike check_report, this does not depend on the order in which the 491 # auditor inserted its findings: one fault can legitimately be reported 492 # against several operations (a coin with a bad denomination signature is 493 # flagged once per operation that used it). 494 function check_report_any() { 495 call_endpoint "$1" 496 NAME=$(echo "$1" | tr '-' '_') 497 # shellcheck disable=SC2086 498 jq -e --arg want "$3" "any(.\"${NAME}\"[]; .\"$2\" == \$want)" \ 499 < "${MY_TMP_DIR}/${1}.json" \ 500 > /dev/null \ 501 || exit_fail "$1::$2 (no entry with value $3)" 502 echo "PASS" 503 } 504 505 # Check that report $1 has at least one entry, without pinning down any of 506 # its fields. Useful when the injected fault is certain to be found but 507 # the exact amounts depend on how much of a reserve happened to be left. 508 function check_some_report() { 509 call_endpoint "$1" 510 NAME=$(echo "$1" | tr '-' '_') 511 # shellcheck disable=SC2086 512 jq -e .\"${NAME}\"[0] \ 513 < "${MY_TMP_DIR}/${1}.json" \ 514 > /dev/null \ 515 || exit_fail "Wanted a report for $1, but got none" 516 echo "PASS" 517 } 518 519 function check_no_report() { 520 call_endpoint "$1" 521 NAME=$(echo "$1" | tr '-' '_') 522 # shellcheck disable=SC2086 523 jq -e .\"${NAME}\"[0] \ 524 < "${MY_TMP_DIR}/${1}.json" \ 525 > /dev/null \ 526 && exit_fail "Wanted empty report for $1, but got incidents" 527 echo "PASS" 528 } 529 530 function check_report_neg() { 531 call_endpoint "$1" 532 NAME=$(echo "$1" | tr '-' '_') 533 # shellcheck disable=SC2086 534 VAL=$(jq -r .\"${NAME}\"[0].\"$2\" < "${MY_TMP_DIR}/${1}.json") 535 if [ "$VAL" == "$3" ] 536 then 537 exit_fail "$1::$2 (got $VAL, wanted $3)" 538 fi 539 echo "PASS" 540 } 541 542 function check_row() { 543 call_endpoint "$1" 544 NAME=$(echo "$1" | tr '-' '_') 545 if [ -n "${3+x}" ] 546 then 547 RID="$2" 548 WANT="$3" 549 else 550 RID="row_id" 551 WANT="$2" 552 fi 553 # shellcheck disable=SC2086 554 ROW=$(jq -r .\"${NAME}\"[0].\"${RID}\" < "${MY_TMP_DIR}/${1}.json") 555 if [ "$ROW" != "$WANT" ] 556 then 557 exit_fail "Row ${1} wrong (got ${ROW}, wanted ${WANT})" 558 fi 559 echo "PASS" 560 } 561 562 563 564 # Amount the exchange wired to the merchant that passed KYC. 565 CLEARED_WIRED="TESTKUDOS:8.02" 566 # Amount the exchange aggregated for the merchant that did not pass KYC 567 # and then parked in aggregation_transient instead of wiring it out. 568 WITHHELD="TESTKUDOS:8.03" 569 # The account of that merchant. The IBAN libeufin derives from the account 570 # name is stable across regenerations of the reference database, so this can be 571 # hardcoded -- but the wire transfer identifier cannot: the aggregator draws it 572 # with GNUNET_CRYPTO_random_block(), so tests 14 and 15 read it out of the 573 # exchange's own tables instead of naming it. 574 BLOCKED_ACCOUNT="payto://iban/DE61109451?receiver-name=Blocked44" 575 # Two KYCAUTH wire transfers of TESTKUDOS:0.1, one per merchant instance. 576 KYCAUTH_IN="TESTKUDOS:0.2" 577 # Everything the exchange's bank account was credited with: TESTKUDOS:56 of 578 # reserve top-ups plus the two KYCAUTH transfers above. The KYCAUTH part is 579 # revenue rather than a liability, but it is money in the account all the 580 # same, so it belongs in the balance the account is expected to have. 581 WIRE_IN="TESTKUDOS:56.2" 582 583 584 # Assert that all of the auditor's loss, delta and irregularity balances 585 # are zero, i.e. that nothing about this database looks wrong to it. 586 function check_all_clean() { 587 echo -n "Test for emergencies... " 588 check_no_report "emergency" 589 echo -n "Test for emergencies by count... " 590 check_no_report "emergency-by-count" 591 echo -n "Test for deposit confirmation problems... " 592 check_no_report "deposit-confirmation" 593 echo -n "Test for denomination key validity problems... " 594 check_no_report "denomination-key-validity-withdraw-inconsistency" 595 echo -n "Test for denominations without signatures... " 596 check_no_report "denominations-without-sigs" 597 echo -n "Test for wire out inconsistencies... " 598 check_no_report "wire-out-inconsistency" 599 echo -n "Test for reserve in inconsistencies... " 600 check_no_report "reserve-in-inconsistency" 601 echo -n "Test for KYCAUTH in inconsistencies... " 602 check_no_report "kycauth-in-inconsistency" 603 echo -n "Test for misattribution inconsistencies... " 604 check_no_report "misattribution-in-inconsistency" 605 echo -n "Test for row inconsistencies... " 606 check_no_report "row-inconsistency" 607 echo -n "Test for minor row inconsistencies... " 608 check_no_report "row-minor-inconsistencies" 609 echo -n "Test for wire format inconsistencies... " 610 check_no_report "wire-format-inconsistency" 611 echo -n "Test for arithmetic inconsistencies... " 612 check_no_report "amount-arithmetic-inconsistency" 613 echo -n "Test for coin inconsistencies... " 614 check_no_report "coin-inconsistency" 615 echo -n "Test for bad signature losses... " 616 check_no_report "bad-sig-losses" 617 echo -n "Test for fee time inconsistencies... " 618 check_no_report "fee-time-inconsistency" 619 echo -n "Test for purses not closed... " 620 check_no_report "purse-not-closed-inconsistencies" 621 echo -n "Test for reserves not closed... " 622 check_no_report "reserve-not-closed-inconsistency" 623 echo -n "Test for insufficient reserve balances... " 624 check_no_report "reserve-balance-insufficient-inconsistency" 625 echo -n "Test for wrong reserve balance summaries... " 626 check_no_report "reserve-balance-summary-wrong-inconsistency" 627 echo -n "Test for closure lags... " 628 check_no_report "closure-lags" 629 630 # Just to test the endpoint and for logging ... 631 call_endpoint "balances" 632 633 echo -n "Testing aggregation bad sig loss balance... " 634 check_balance \ 635 "aggregation_total_bad_sig_loss" \ 636 "TESTKUDOS:0" \ 637 "Wrong total bad sig loss from aggregation" 638 echo -n "Testing coin irregular loss balance... " 639 check_balance \ 640 "coin_irregular_loss" \ 641 "TESTKUDOS:0" \ 642 "Wrong total irregular loss from coins" 643 echo -n "Testing reserves bad sig loss balance... " 644 check_balance \ 645 "reserves_total_bad_sig_loss" \ 646 "TESTKUDOS:0" \ 647 "Wrong total bad sig loss from reserves" 648 echo -n "Testing purse bad sig loss balance... " 649 check_balance \ 650 "purse_total_bad_sig_loss" \ 651 "TESTKUDOS:0" \ 652 "Wrong total bad sig loss from purses" 653 654 echo -n "Test for bad incoming delta plus... " 655 check_balance \ 656 "total_bad_amount_in_plus" \ 657 "TESTKUDOS:0" \ 658 "Expected total wire in delta plus wrong" 659 echo -n "Test for bad incoming delta minus... " 660 check_balance \ 661 "total_bad_amount_in_minus" \ 662 "TESTKUDOS:0" \ 663 "Expected total wire in delta minus wrong" 664 echo -n "Test for bad outgoing delta plus... " 665 check_balance \ 666 "total_bad_amount_out_plus" \ 667 "TESTKUDOS:0" \ 668 "Expected total wire out delta plus wrong" 669 echo -n "Test for bad outgoing delta minus... " 670 check_balance \ 671 "total_bad_amount_out_minus" \ 672 "TESTKUDOS:0" \ 673 "Expected total wire out delta minus wrong" 674 echo -n "Test for misattribution amounts... " 675 check_balance \ 676 "total_misattribution_in" \ 677 "TESTKUDOS:0" \ 678 "Expected total misattribution in wrong" 679 680 echo -n "Test for aggregation wire out delta plus... " 681 check_balance \ 682 "aggregation_total_wire_out_delta_plus" \ 683 "TESTKUDOS:0" \ 684 "Expected total wire out delta plus wrong" 685 echo -n "Test for aggregation wire out delta minus... " 686 check_balance \ 687 "aggregation_total_wire_out_delta_minus" \ 688 "TESTKUDOS:0" \ 689 "Expected total wire out delta minus wrong" 690 691 echo -n "Checking for unexpected aggregation delta plus... " 692 check_balance \ 693 "aggregation_total_arithmetic_delta_plus" \ 694 "TESTKUDOS:0" \ 695 "Wrong arithmetic delta plus from aggregations" 696 echo -n "Checking for unexpected aggregation delta minus... " 697 check_balance \ 698 "aggregation_total_arithmetic_delta_minus" \ 699 "TESTKUDOS:0" \ 700 "Wrong arithmetic delta minus from aggregations" 701 echo -n "Checking for unexpected coin delta plus... " 702 check_balance \ 703 "coins_total_arithmetic_delta_plus" \ 704 "TESTKUDOS:0" \ 705 "Wrong arithmetic delta plus from coins" 706 echo -n "Checking for unexpected coin delta minus... " 707 check_balance \ 708 "coins_total_arithmetic_delta_minus" \ 709 "TESTKUDOS:0" \ 710 "Wrong arithmetic delta minus from coins" 711 echo -n "Checking for unexpected reserves delta plus... " 712 check_balance \ 713 "reserves_total_arithmetic_delta_plus" \ 714 "TESTKUDOS:0" \ 715 "Wrong arithmetic delta plus from reserves" 716 echo -n "Checking for unexpected reserves delta minus... " 717 check_balance \ 718 "reserves_total_arithmetic_delta_minus" \ 719 "TESTKUDOS:0" \ 720 "Wrong arithmetic delta minus from reserves" 721 } 722 723 724 # The KYC-specific assertions: how much arrived by KYCAUTH, how much left 725 # the exchange, and how much it is sitting on. 726 function check_kyc_state() { 727 echo -n "Checking the KYCAUTH credits were accounted for... " 728 check_balance \ 729 "total_kycauth_in" \ 730 "$KYCAUTH_IN" \ 731 "Wrong total for KYCAUTH wire transfers" 732 733 # KYCAUTH money is not credited to a reserve and never paid back, so 734 # the auditor books it as revenue of the exchange operator and counts 735 # it towards the balance the bank account is expected to have. 736 echo -n "Checking the KYCAUTH credits were booked as revenue... " 737 check_balance \ 738 "total_kycauth_revenue" \ 739 "$KYCAUTH_IN" \ 740 "Wrong KYCAUTH revenue" 741 echo -n "Checking the KYCAUTH credits reached the balance sheet... " 742 check_balance \ 743 "total_wire_in" \ 744 "$WIRE_IN" \ 745 "Wrong total credited to the exchange" 746 747 echo -n "Checking that only the cleared merchant was paid... " 748 check_balance \ 749 "total_wire_out" \ 750 "$CLEARED_WIRED" \ 751 "Wrong total wired out" 752 753 # The exchange aggregated the blocked merchant's coins, wrote the 754 # aggregation_tracking rows and then parked the payout in 755 # aggregation_transient rather than executing it. Confirm from the 756 # exchange's own database that the money really is still there, so 757 # that the auditor-side assertion below is about the auditor and not 758 # about the reference database having drifted. 759 echo -n "Checking that the exchange withheld the blocked payout... " 760 TRANSIENT=$(psql -Aqt "$DB" \ 761 -c "SELECT COUNT(*) FROM exchange.aggregation_transient;") 762 if [ "$TRANSIENT" != "1" ] 763 then 764 exit_fail "Expected one withheld aggregation, got ${TRANSIENT}" 765 fi 766 WIRED=$(psql -Aqt "$DB" -c "SELECT COUNT(*) FROM exchange.wire_out;") 767 if [ "$WIRED" != "1" ] 768 then 769 exit_fail "Expected exactly one wire transfer, got ${WIRED}" 770 fi 771 echo "PASS" 772 773 # The exchange also said *why*, in the append-only table that replication 774 # can carry. aggregation_transient above is the same claim, but an 775 # external auditor never gets to see it, so this is the row the auditor's 776 # classification actually rests on. Reason 2 is 777 # TALER_EXCHANGEDB_DR_KYC. 778 # 779 # Restrict this to transfers that have not gone out. Both merchants were 780 # withheld on the generator's first aggregator run -- neither had passed 781 # KYC yet -- so the cleared one has a deferral of its own, made obsolete by 782 # the wire_out row that followed. Append-only means those stay on file, 783 # which is the point: the record of a hold outlives the hold. 784 echo -n "Checking that the exchange recorded why it withheld it... " 785 DEFERRED=$(psql -Aqt "$DB" \ 786 -c "SELECT COUNT(*) FROM exchange.aggregation_deferrals d 787 WHERE d.deferral_reason=2 788 AND d.legitimization_requirement_serial_id != 0 789 AND NOT EXISTS ( 790 SELECT 1 FROM exchange.wire_out w 791 WHERE w.wtid_raw=d.wtid_raw);") 792 if [ "$DEFERRED" != "1" ] 793 then 794 exit_fail "Expected one open KYC deferral on record, got ${DEFERRED}" 795 fi 796 echo "PASS" 797 798 # ... and the auditor sees it. The withheld deposits do carry 799 # aggregation_tracking rows, so they are not "lagging" in the sense of 800 # total_amount_lag, which counts deposits the exchange has not aggregated 801 # at all; that balance is legitimately zero here. What the exchange has 802 # not done is the wire transfer, and that is what total_aml_hold counts. 803 echo -n "Checking that nothing is waiting to be aggregated... " 804 check_balance \ 805 "total_amount_lag" \ 806 "TESTKUDOS:0" \ 807 "Wrong total for deposits that were never aggregated" 808 echo -n "Checking that the withheld payout is booked as an AML hold... " 809 check_balance \ 810 "total_aml_hold" \ 811 "$WITHHELD" \ 812 "Wrong total withheld pending legitimization" 813 # The blocked merchant is the *only* transfer the exchange owes, and its 814 # KYC measure is open, so nothing may land in the unexplained bucket. 815 # Were the two ever to swap, the exchange would be stalling a payout it 816 # has no legal reason to stall, which is the case this split exists for. 817 echo -n "Checking that no payout is unexplained... " 818 check_balance \ 819 "total_transfer_lag" \ 820 "TESTKUDOS:0" \ 821 "Exchange is withholding funds it gave no reason for" 822 # Every deposit here is aggregated into a payout worth making, so the 823 # third bucket -- money parked only because the aggregate would not 824 # cover its wire fee -- must be empty. It exists so that such money 825 # does not read as either of the two above, both of which say something 826 # is wrong. 827 echo -n "Checking that nothing is parked as a small aggregate... " 828 check_balance \ 829 "total_small_aggregate" \ 830 "TESTKUDOS:0" \ 831 "Exchange parked an aggregate as too small to pay out" 832 echo -n "Checking that the hold names the blocked account... " 833 check_report_any \ 834 "aml-holds" \ 835 "account" \ 836 "$BLOCKED_ACCOUNT" 837 echo -n "Checking that the hold names the amount... " 838 check_report "aml-holds" "amount" "$WITHHELD" 839 echo -n "Checking that the hold is attributed to KYC... " 840 check_report "aml-holds" "deferral_reason" "KYC" 841 echo -n "Test for pending deposits... " 842 check_no_report "pending-deposits" 843 echo -n "Test for early aggregations... " 844 check_no_report "early-aggregation" 845 } 846 847 848 # The KYC artefacts must not make the auditor invent findings. 849 function test_0() { 850 851 echo "===========0: normal run with aggregator===========" 852 run_audit aggregator 853 check_auditor_running 854 855 echo "Checking output" 856 check_all_clean 857 check_kyc_state 858 859 # cannot easily undo aggregator, hence full reload 860 full_reload 861 cleanup 862 } 863 864 865 # Same, but without letting the aggregator run first. The aggregator 866 # cannot lawfully do anything more than it already did when the reference 867 # database was generated, so the auditor's view must be identical. 868 function test_1() { 869 870 echo "===========1: normal run===========" 871 run_audit 872 check_auditor_running 873 874 echo "Checking output" 875 check_all_clean 876 check_kyc_state 877 878 # Database was unmodified, no need to undo 879 echo "OK" 880 } 881 882 883 # A KYCAUTH credit that the bank does not agree is a KYCAUTH credit. 884 function test_2() { 885 886 echo "===========2: KYCAUTH credit reclassified by the bank===========" 887 # Modifying the bank's books needs libeufin to be restarted, it does 888 # not re-read them. 889 pre_audit 890 stop_libeufin 891 echo -n "Modifying database: " 892 # libeufin records for every credit to the exchange whether it was a 893 # reserve top-up or an account authorisation. Turn one of the two 894 # KYCAUTH credits into a reserve credit: the KYCAUTH total the auditor 895 # tracks must drop by that amount, and because the 32 bytes of KYCAUTH 896 # metadata are not a reserve public key the exchange knows, the credit 897 # now also looks like an incoming transfer the exchange never booked. 898 echo "UPDATE libeufin_bank.taler_exchange_incoming SET type='reserve' WHERE exchange_incoming_id=(SELECT MIN(exchange_incoming_id) FROM libeufin_bank.taler_exchange_incoming WHERE type='kyc')" \ 899 | psql -Aqt "$DB" 900 echo "DONE" 901 launch_libeufin 902 await_bank 903 904 audit_only 905 check_auditor_running 906 907 echo -n "Checking that the KYCAUTH total dropped... " 908 check_balance \ 909 "total_kycauth_in" \ 910 "TESTKUDOS:0.1" \ 911 "Wrong total for KYCAUTH wire transfers" 912 # Only the bank's books were touched, so the exchange still claims the 913 # KYCAUTH revenue for a transfer the bank no longer reports as one. 914 echo -n "Checking that the exchange still claims the revenue... " 915 check_balance \ 916 "total_kycauth_revenue" \ 917 "$KYCAUTH_IN" \ 918 "Wrong KYCAUTH revenue" 919 echo -n "Checking that the unbooked credit was flagged... " 920 check_report_any \ 921 "reserve-in-inconsistency" \ 922 "diagnostic" "unknown to exchange" 923 echo -n "Checking that it was counted as an excess credit... " 924 check_balance \ 925 "total_bad_amount_in_plus" \ 926 "TESTKUDOS:0.1" \ 927 "Wrong total_bad_amount_in_plus" 928 929 # ... and the other side of the same disagreement: the exchange has a 930 # kycauths_in row for a transfer the bank now says was something else, 931 # so the auditor must not take the exchange's word for it. 932 echo -n "Checking that the unconfirmed KYCAUTH was flagged... " 933 check_report_any \ 934 "kycauth-in-inconsistency" \ 935 "diagnostic" "KYCAUTH wire transfer claimed by exchange not found" 936 echo -n "Checking the amount the exchange claims... " 937 check_report_any \ 938 "kycauth-in-inconsistency" \ 939 "amount_exchange_expected" "TESTKUDOS:0.1" 940 echo -n "Checking that it was counted as a missing credit... " 941 check_balance \ 942 "total_bad_amount_in_minus" \ 943 "TESTKUDOS:0.1" \ 944 "Wrong total_bad_amount_in_minus" 945 946 post_audit 947 full_reload 948 cleanup 949 } 950 951 952 # The rest of the tests inject the same kinds of faults test-auditor.sh 953 # injects, to show that KYC does not stop the auditor from finding them. 954 955 956 # Change the amount of an incoming wire transfer as recorded by the 957 # exchange. 958 function test_3() { 959 960 echo "===========3: reserves_in inconsistency===========" 961 echo -n "Modifying database: " 962 echo "UPDATE exchange.reserves_in SET credit.val=5 WHERE reserve_in_serial_id=1" \ 963 | psql -Aqt "$DB" 964 echo "DONE" 965 966 run_audit 967 check_auditor_running 968 969 echo -n "Testing inconsistency detection... " 970 check_report \ 971 "reserve-in-inconsistency" \ 972 "row_id" 1 973 echo -n "Testing amount wired... " 974 check_report \ 975 "reserve-in-inconsistency" \ 976 "amount_wired" "TESTKUDOS:10" 977 echo -n "Testing amount expected... " 978 check_report \ 979 "reserve-in-inconsistency" \ 980 "amount_exchange_expected" "TESTKUDOS:5" 981 echo -n "Checking wire credit balance plus... " 982 check_balance \ 983 "total_bad_amount_in_plus" \ 984 "TESTKUDOS:5" \ 985 "Expected total_bad_amount_in_plus wrong" 986 987 full_reload 988 cleanup 989 } 990 991 992 # Test where denom_sig in the known_coins table is wrong (=> bad signature) 993 function test_4() { 994 995 echo "===========4: known_coins signature wrong===========" 996 OLD_ROW=$(echo "SELECT known_coin_id FROM exchange.known_coins LIMIT 1;" | psql "$DB" -Aqt) 997 COIN_PUB=$(echo "SELECT coin_pub FROM exchange.known_coins WHERE known_coin_id=$OLD_ROW;" | psql "$DB" -Aqt) 998 # shellcheck disable=SC2028 999 echo "UPDATE exchange.known_coins SET denom_sig='\x0000000100000000287369672d76616c200a2028727361200a2020287320233542383731423743393036444643303442424430453039353246413642464132463537303139374131313437353746324632323332394644443146324643333445393939413336363430334233413133324444464239413833353833464536354442374335434445304441453035374438363336434541423834463843323843344446304144363030343430413038353435363039373833434431333239393736423642433437313041324632414132414435413833303432434346314139464635394244434346374436323238344143354544364131373739463430353032323241373838423837363535453434423145443831364244353638303232413123290a2020290a20290b' WHERE coin_pub='$COIN_PUB'" \ 1000 | psql -Aqt "$DB" 1001 1002 run_audit 1003 check_auditor_running 1004 1005 echo -n "Checking bad-signature-loss detected... " 1006 check_report_neg \ 1007 "bad-sig-losses" \ 1008 "loss" "TESTKUDOS:0" 1009 echo -n "Checking bad-signature-loss balance update... " 1010 check_not_balance \ 1011 "coin_irregular_loss" \ 1012 "TESTKUDOS:0" \ 1013 "Wrong total bad sig loss" 1014 1015 full_reload 1016 cleanup 1017 } 1018 1019 1020 # Test where the amount the bank actually wired differs from what the 1021 # exchange claims. 1022 function test_5() { 1023 1024 echo "===========5: incorrect wire_out amount===========" 1025 pre_audit 1026 stop_libeufin 1027 echo -n "Modifying database: " 1028 echo "UPDATE libeufin_bank.bank_account_transactions SET amount=(50,0) WHERE debtor_name='Exchange Company';" \ 1029 | psql -Aqt "$DB" 1030 echo "DONE" 1031 launch_libeufin 1032 await_bank 1033 1034 audit_only 1035 check_auditor_running 1036 1037 echo -n "Testing wire-out-inconsistency expected... " 1038 check_report \ 1039 "wire-out-inconsistency" \ 1040 "expected" \ 1041 "$CLEARED_WIRED" 1042 echo -n "Testing wire-out-inconsistency claimed... " 1043 check_report \ 1044 "wire-out-inconsistency" \ 1045 "claimed" \ 1046 "TESTKUDOS:50" 1047 echo -n "Testing bad_amount_out_plus balance reporting... " 1048 check_not_balance \ 1049 "total_bad_amount_out_plus" \ 1050 "TESTKUDOS:0" \ 1051 "reported total_bad_amount_out_plus wrong" 1052 1053 post_audit 1054 full_reload 1055 cleanup 1056 } 1057 1058 1059 # Test where the exchange claims to have received more than the bank sent, 1060 # which also throws its reserve balance summary off. 1061 function test_6() { 1062 1063 echo "===========6: reserve balance summary wrong===========" 1064 echo -n "Modifying database: " 1065 echo "UPDATE exchange.reserves_in SET credit.val=15 WHERE reserve_in_serial_id=1" \ 1066 | psql -Aqt "$DB" 1067 echo "DONE" 1068 1069 run_audit 1070 check_auditor_running 1071 1072 echo -n "Checking reserve balance summary inconsistency detected... " 1073 check_some_report \ 1074 "reserve-balance-summary-wrong-inconsistency" 1075 echo -n "Testing amount the exchange expected... " 1076 check_report \ 1077 "reserve-in-inconsistency" \ 1078 "amount_exchange_expected" "TESTKUDOS:15" 1079 echo -n "Checking wire credit balance minus... " 1080 check_balance \ 1081 "total_bad_amount_in_minus" \ 1082 "TESTKUDOS:5" \ 1083 "Wrong total_bad_amount_in_minus" 1084 1085 full_reload 1086 cleanup 1087 } 1088 1089 1090 # Test where the exchange forgot a deposit that a merchant can prove. 1091 function test_7() { 1092 1093 echo "===========7: deposit wire target malformed===========" 1094 SERIAL=$(echo "SELECT batch_deposit_serial_id FROM exchange.coin_deposits WHERE (amount_with_fee).val=3 ORDER BY batch_deposit_serial_id LIMIT 1" | psql "$DB" -Aqt) 1095 # shellcheck disable=SC2028 1096 echo "INSERT INTO exchange.wire_targets (payto_uri, wire_target_h_payto) VALUES ('payto://x-taler-bank/localhost/testuser-xxlargtp', '\x1e8f31936b3cee8f8afd3aac9e38b5db42d45b721ffc4eb1e5b9ddaf1565660b');" \ 1097 | psql "$DB" -Aqt 1098 # shellcheck disable=SC2028 1099 echo "UPDATE exchange.batch_deposits SET wire_target_h_payto='\x1e8f31936b3cee8f8afd3aac9e38b5db42d45b721ffc4eb1e5b9ddaf1565660b' WHERE batch_deposit_serial_id=${SERIAL};" \ 1100 | psql -Aqt "$DB" 1101 1102 run_audit 1103 check_auditor_running 1104 1105 echo -n "Checking correct operation of loss reported... " 1106 check_report \ 1107 "bad-sig-losses" \ 1108 "operation" "deposit" 1109 echo -n "Checking correct problem row ID reported... " 1110 check_report \ 1111 "bad-sig-losses" \ 1112 "problem_row_id" "$SERIAL" 1113 echo -n "Checking irregular coin loss... " 1114 check_not_balance \ 1115 "coin_irregular_loss" \ 1116 "TESTKUDOS:0" \ 1117 "Wrong total irregular coin loss" 1118 1119 full_reload 1120 cleanup 1121 } 1122 1123 1124 # The reverse of test 2: the bank reports a KYCAUTH transfer that the 1125 # exchange has no record of. An exchange that could make such a row 1126 # disappear could also make the account authorisation it justified 1127 # disappear, so this must be flagged even though the money involved is 1128 # small. 1129 function test_8() { 1130 1131 echo "===========8: KYCAUTH credit missing from the exchange===========" 1132 echo -n "Modifying database: " 1133 echo "DELETE FROM exchange.kycauths_in WHERE kycauth_in_serial_id=1" \ 1134 | psql -Aqt "$DB" 1135 echo "DONE" 1136 1137 run_audit 1138 check_auditor_running 1139 1140 echo -n "Checking that the bank's view is unchanged... " 1141 check_balance \ 1142 "total_kycauth_in" \ 1143 "$KYCAUTH_IN" \ 1144 "Wrong total for KYCAUTH wire transfers" 1145 echo -n "Checking that the exchange claims less revenue... " 1146 check_balance \ 1147 "total_kycauth_revenue" \ 1148 "TESTKUDOS:0.1" \ 1149 "Wrong KYCAUTH revenue" 1150 echo -n "Checking that the unbooked KYCAUTH was flagged... " 1151 check_report_any \ 1152 "kycauth-in-inconsistency" \ 1153 "diagnostic" "unknown to exchange" 1154 echo -n "Checking the amount the bank reports... " 1155 check_report_any \ 1156 "kycauth-in-inconsistency" \ 1157 "amount_wired" "TESTKUDOS:0.1" 1158 echo -n "Checking that it was counted as an excess credit... " 1159 check_balance \ 1160 "total_bad_amount_in_plus" \ 1161 "TESTKUDOS:0.1" \ 1162 "Wrong total_bad_amount_in_plus" 1163 1164 full_reload 1165 cleanup 1166 } 1167 1168 1169 # The exchange booked a KYCAUTH transfer, but for a different amount than 1170 # the bank actually moved. 1171 function test_9() { 1172 1173 echo "===========9: KYCAUTH amount inconsistency===========" 1174 echo -n "Modifying database: " 1175 echo "UPDATE exchange.kycauths_in SET credit.val=5 WHERE kycauth_in_serial_id=1" \ 1176 | psql -Aqt "$DB" 1177 echo "DONE" 1178 1179 run_audit 1180 check_auditor_running 1181 1182 echo -n "Checking that the mismatch was flagged... " 1183 check_report_any \ 1184 "kycauth-in-inconsistency" \ 1185 "diagnostic" "wire amount does not match" 1186 echo -n "Checking the amount the exchange claims... " 1187 check_report_any \ 1188 "kycauth-in-inconsistency" \ 1189 "amount_exchange_expected" "TESTKUDOS:5.1" 1190 echo -n "Checking the amount the bank reports... " 1191 check_report_any \ 1192 "kycauth-in-inconsistency" \ 1193 "amount_wired" "TESTKUDOS:0.1" 1194 echo -n "Checking that the shortfall was counted... " 1195 check_balance \ 1196 "total_bad_amount_in_minus" \ 1197 "TESTKUDOS:5" \ 1198 "Wrong total_bad_amount_in_minus" 1199 echo -n "Checking that the inflated revenue reached the balance sheet... " 1200 check_balance \ 1201 "total_kycauth_revenue" \ 1202 "TESTKUDOS:5.2" \ 1203 "Wrong KYCAUTH revenue" 1204 1205 full_reload 1206 cleanup 1207 } 1208 1209 1210 # The auditor may well see a KYCAUTH transfer at the bank before the 1211 # exchange has booked it. It reports that, and must take the report back 1212 # -- together with the excess credit it booked alongside -- once the 1213 # exchange catches up. Without the retraction of the balance, a purely 1214 # transient disagreement would inflate total_bad_amount_in_plus forever. 1215 function test_10() { 1216 1217 echo "===========10: KYCAUTH finding retracted when the exchange catches up===========" 1218 echo -n "Modifying database: " 1219 # Hide the exchange's record of the *last* KYCAUTH transfer, keeping a 1220 # copy so it can be restored byte for byte. It has to be the last one: 1221 # kycauth_in_cb() resumes from a progress point, so a row reinstated 1222 # below that point would never be looked at again. 1223 echo "CREATE TABLE stashed_kycauth AS 1224 SELECT * FROM exchange.kycauths_in 1225 WHERE kycauth_in_serial_id 1226 = (SELECT MAX(kycauth_in_serial_id) FROM exchange.kycauths_in); 1227 DELETE FROM exchange.kycauths_in 1228 WHERE kycauth_in_serial_id 1229 = (SELECT MAX(kycauth_in_serial_id) FROM exchange.kycauths_in);" \ 1230 | psql -Aqt "$DB" 1231 echo "DONE" 1232 1233 pre_audit 1234 audit_only 1235 check_auditor_running 1236 1237 echo -n "Checking that the unbooked transfer was reported... " 1238 check_report_any \ 1239 "kycauth-in-inconsistency" \ 1240 "diagnostic" "unknown to exchange" 1241 echo -n "Checking that it was counted as an excess credit... " 1242 check_balance \ 1243 "total_bad_amount_in_plus" \ 1244 "TESTKUDOS:0.1" \ 1245 "Wrong total_bad_amount_in_plus" 1246 1247 # The exchange catches up. No progress points are touched: the credit 1248 # audit is simply run once more, exactly as a resident helper would on 1249 # its next wake-up. 1250 echo -n "Letting the exchange catch up: " 1251 echo "INSERT INTO exchange.kycauths_in SELECT * FROM stashed_kycauth; 1252 DROP TABLE stashed_kycauth;" \ 1253 | psql -Aqt "$DB" 1254 echo "DONE" 1255 1256 echo -n "Re-running the credit audit ..." 1257 $VALGRIND taler-helper-auditor-wire-credit \ 1258 -i \ 1259 -L DEBUG \ 1260 -c "$CONF" \ 1261 -t \ 1262 > "${MY_TMP_DIR}/test-audit-wire-credit-retract.out" \ 1263 2> "${MY_TMP_DIR}/test-audit-wire-credit-retract.err" \ 1264 || exit_fail "wire credit re-audit failed (see ${MY_TMP_DIR}/test-audit-wire-credit-retract.*)" 1265 echo " DONE" 1266 1267 echo -n "Checking that the report was retracted... " 1268 check_no_report "kycauth-in-inconsistency" 1269 echo -n "Checking that the excess credit was retracted too... " 1270 check_balance \ 1271 "total_bad_amount_in_plus" \ 1272 "TESTKUDOS:0" \ 1273 "Retracted finding left total_bad_amount_in_plus inflated" 1274 1275 post_audit 1276 full_reload 1277 cleanup 1278 } 1279 1280 1281 # An AML staff appointment that the exchange's offline master key did not 1282 # sign. Without this check an exchange could appoint an AML officer, or 1283 # quietly widen a read-only officer to read-write, with the offline key never 1284 # having been involved. 1285 function test_11() { 1286 1287 echo "===========11: AML staff appointment with a bad master signature===========" 1288 echo -n "Modifying database: " 1289 # shellcheck disable=SC2028 1290 echo "INSERT INTO exchange.aml_staff 1291 (decider_pub, master_sig, decider_name, is_active, read_only, last_change) 1292 VALUES 1293 ('\x1111111111111111111111111111111111111111111111111111111111111111', 1294 '\x22222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222', 1295 'Unappointed Officer', TRUE, FALSE, 1000000000000000);" \ 1296 | psql -Aqt "$DB" 1297 echo "DONE" 1298 1299 run_audit 1300 check_auditor_running 1301 1302 echo -n "Checking that the appointment was flagged... " 1303 check_report_any \ 1304 "row-inconsistency" \ 1305 "diagnostic" "invalid master key signature on staff status" 1306 echo -n "Checking that it was blamed on the right table... " 1307 check_report_any \ 1308 "row-inconsistency" \ 1309 "row_table" "aml_staff" 1310 1311 # aml_staff is append-only, so the helper can and must resume from where 1312 # it stopped instead of re-reading the whole table on every wake-up. 1313 # Both halves have to hold: the cursor has to be stored at all, and the 1314 # second run has to pick it up. 1315 echo -n "Checking that the aml_staff cursor was stored... " 1316 AML_PP=$(psql -Aqt "$DB" \ 1317 -c "SELECT progress_offset FROM auditor.auditor_progress \ 1318 WHERE progress_key='aml_staff_uuid';") 1319 if [ -z "$AML_PP" ] || [ "$AML_PP" = "0" ] 1320 then 1321 exit_fail "aml_staff_uuid progress point not recorded (got '$AML_PP')" 1322 fi 1323 echo "PASS ($AML_PP)" 1324 echo -n "Checking that the second run resumed from it... " 1325 if ! grep -q "Resuming AML audit at $AML_PP/" \ 1326 "${MY_TMP_DIR}/test-audit-aml-inc.err" 1327 then 1328 exit_fail "incremental AML audit did not resume at $AML_PP" 1329 fi 1330 echo "PASS" 1331 1332 full_reload 1333 cleanup 1334 } 1335 1336 1337 # An AML decision that no valid officer signature backs, by somebody the 1338 # exchange never appointed at all. Both halves have to be reported: the 1339 # signature is what makes the decision accountable, and the appointment is 1340 # what makes the signature mean anything. 1341 function test_12() { 1342 1343 echo "===========12: AML decision by an unappointed officer===========" 1344 echo -n "Modifying database: " 1345 # shellcheck disable=SC2028 1346 echo "INSERT INTO exchange.aml_history 1347 (h_payto, justification, decider_pub, decider_sig, outcome_serial_id) 1348 SELECT h_payto, 1349 'injected by test 12', 1350 '\x3333333333333333333333333333333333333333333333333333333333333333', 1351 '\x44444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444', 1352 outcome_serial_id 1353 FROM exchange.legitimization_outcomes 1354 ORDER BY outcome_serial_id 1355 LIMIT 1;" \ 1356 | psql -Aqt "$DB" 1357 echo "DONE" 1358 1359 run_audit 1360 check_auditor_running 1361 1362 echo -n "Checking that the bad signature was flagged... " 1363 check_report_any \ 1364 "row-inconsistency" \ 1365 "diagnostic" "invalid officer signature on decision" 1366 echo -n "Checking that the missing appointment was flagged... " 1367 check_report_any \ 1368 "row-inconsistency" \ 1369 "diagnostic" "officer was not appointed when the decision was made" 1370 echo -n "Checking that it was blamed on the right table... " 1371 check_report_any \ 1372 "row-inconsistency" \ 1373 "row_table" "aml_history" 1374 1375 full_reload 1376 cleanup 1377 } 1378 1379 1380 # `legitimization_outcomes' is the table that says which KYC rules apply to 1381 # an account, so a row nothing produced is how an exchange would quietly 1382 # exempt a customer from KYC. The account here is fresh: it has no AML 1383 # decision pointing at the outcome, no legitimization process, and no earlier 1384 # outcome that expired, which is all three of the ways the exchange's own 1385 # code creates one. 1386 function test_13() { 1387 1388 echo "===========13: legitimization outcome without a justification===========" 1389 echo -n "Modifying database: " 1390 # shellcheck disable=SC2028 1391 echo "INSERT INTO exchange.kyc_targets 1392 (h_normalized_payto, is_wallet) 1393 VALUES 1394 ('\x5555555555555555555555555555555555555555555555555555555555555555', 1395 FALSE); 1396 INSERT INTO exchange.legitimization_outcomes 1397 (h_payto, decision_time, expiration_time, to_investigate) 1398 VALUES 1399 ('\x5555555555555555555555555555555555555555555555555555555555555555', 1400 1000000000000000, 2000000000000000, FALSE);" \ 1401 | psql -Aqt "$DB" 1402 echo "DONE" 1403 1404 run_audit 1405 check_auditor_running 1406 1407 echo -n "Checking that the unjustified outcome was flagged... " 1408 check_report_any \ 1409 "row-inconsistency" \ 1410 "diagnostic" \ 1411 "KYC rules in force without an AML decision, a legitimization process or an expired predecessor" 1412 echo -n "Checking that it was blamed on the right table... " 1413 check_report_any \ 1414 "row-inconsistency" \ 1415 "row_table" "legitimization_outcomes" 1416 1417 # Only the injected row is bad. The outcomes the reference database 1418 # earned honestly are each accounted for by a legitimization process or 1419 # an AML decision, and reporting those too would make the check useless. 1420 echo -n "Checking that the honest outcomes were left alone... " 1421 BAD=$(jq -r '[.row_inconsistency[] 1422 | select(.row_table == "legitimization_outcomes")] 1423 | length' \ 1424 < "${MY_TMP_DIR}/row-inconsistency.json") 1425 if [ "$BAD" != "1" ] 1426 then 1427 exit_fail "wanted exactly 1 bad legitimization outcome, got $BAD" 1428 fi 1429 echo "PASS" 1430 1431 full_reload 1432 cleanup 1433 } 1434 1435 1436 # Releasing the funds must retract the alert, not just stop adding to it. 1437 # The exchange announces the release by writing the `wire_out' row, which is 1438 # the only thing that distinguishes "paid" from "decided what to pay"; so 1439 # that is what this test injects. Nothing else changes: the deposits, the 1440 # aggregation_tracking rows and the open KYC measure all stay exactly as they 1441 # were, which is the point -- the hold must end because the money moved, and 1442 # for no other reason. 1443 function test_14() { 1444 1445 echo "===========14: releasing a withheld payout===========" 1446 echo -n "Modifying database: " 1447 # Take the transfer straight from the transient the exchange parked it in, 1448 # so that the test does not have to know the randomly drawn wtid. 1449 echo "INSERT INTO exchange.wire_out 1450 (execution_date, wtid_raw, wire_target_h_payto, 1451 exchange_account_section, amount) 1452 SELECT 1785869200000000 1453 ,t.wtid_raw 1454 ,t.wire_target_h_payto 1455 ,t.exchange_account_section 1456 ,t.amount 1457 FROM exchange.aggregation_transient t;" \ 1458 | psql -Aqt "$DB" 1459 echo "DONE" 1460 1461 run_audit 1462 check_auditor_running 1463 1464 echo -n "Checking that the hold was retracted... " 1465 check_no_report "aml-holds" 1466 echo -n "Checking that the AML total went back to zero... " 1467 check_balance \ 1468 "total_aml_hold" \ 1469 "TESTKUDOS:0" \ 1470 "Released funds are still counted as withheld" 1471 echo -n "Checking that they did not turn into a lag instead... " 1472 check_balance \ 1473 "total_transfer_lag" \ 1474 "TESTKUDOS:0" \ 1475 "Released funds were reclassified rather than dropped" 1476 echo -n "Checking that they were not parked as a small aggregate... " 1477 check_balance \ 1478 "total_small_aggregate" \ 1479 "TESTKUDOS:0" \ 1480 "Released funds were reclassified rather than dropped" 1481 1482 full_reload 1483 cleanup 1484 } 1485 1486 1487 # What the exchange *claims* decides which balance a hold lands in; whether the 1488 # claim holds up is a separate question, and this is the test that they are 1489 # separate. Here the measure the exchange named is carried through to a 1490 # decision while the exchange goes on withholding the money and goes on saying 1491 # it is withholding it for that measure. 1492 # 1493 # The money must therefore stay in `total_aml_hold' -- the auditor does not get 1494 # to overrule the exchange about its own reasons -- and the auditor must report 1495 # a row_inconsistency against `aggregation_deferrals', because the one part of 1496 # the claim it can check has stopped being true. Deciding what to make of that 1497 # is a job for the humans reading the finding. 1498 function test_15() { 1499 1500 echo "===========15: a hold whose KYC measure was satisfied===========" 1501 echo -n "Modifying database: " 1502 # Carry the measure that is blocking the payout through to a decision, the 1503 # way the exchange would have if the merchant had filled in the form: a 1504 # process referring to the measure, and an outcome for the account decided 1505 # no earlier than that process started. `decision_time' is rounded up to a 1506 # whole second because the exchange stores it rounded and the AML helper 1507 # reads it as a timestamp. 1508 echo "INSERT INTO exchange.legitimization_processes 1509 (h_payto, start_time, expiration_time, provider_name, 1510 legitimization_measure_serial_id, measure_index) 1511 SELECT kt.h_normalized_payto 1512 ,lm.start_time 1513 ,0 1514 ,'full_name_and_birthdate' 1515 ,lm.legitimization_measure_serial_id 1516 ,0 1517 FROM exchange.aggregation_transient t 1518 JOIN exchange.wire_targets wt 1519 ON (wt.wire_target_h_payto=t.wire_target_h_payto) 1520 JOIN exchange.kyc_targets kt 1521 ON (kt.h_normalized_payto=wt.h_normalized_payto) 1522 JOIN exchange.legitimization_measures lm 1523 USING (access_token); 1524 INSERT INTO exchange.legitimization_outcomes 1525 (h_payto, decision_time, expiration_time, to_investigate) 1526 SELECT kt.h_normalized_payto 1527 ,(lm.start_time/1000000+1)*1000000 1528 ,2000000000000000 1529 ,FALSE 1530 FROM exchange.aggregation_transient t 1531 JOIN exchange.wire_targets wt 1532 ON (wt.wire_target_h_payto=t.wire_target_h_payto) 1533 JOIN exchange.kyc_targets kt 1534 ON (kt.h_normalized_payto=wt.h_normalized_payto) 1535 JOIN exchange.legitimization_measures lm 1536 USING (access_token);" \ 1537 | psql -Aqt "$DB" 1538 echo "DONE" 1539 1540 run_audit 1541 check_auditor_running 1542 1543 echo -n "Checking that the exchange is still taken at its word... " 1544 check_balance \ 1545 "total_aml_hold" \ 1546 "$WITHHELD" \ 1547 "Auditor reclassified a hold the exchange still attributes to KYC" 1548 echo -n "Checking that it did not become an unexplained lag... " 1549 check_balance \ 1550 "total_transfer_lag" \ 1551 "TESTKUDOS:0" \ 1552 "Auditor overruled the exchange's stated reason" 1553 echo -n "Checking that the hold still names the claimed reason... " 1554 check_report "aml-holds" "deferral_reason" "KYC" 1555 echo -n "Checking that it still names the blocked account... " 1556 check_report_any \ 1557 "aml-holds" \ 1558 "account" \ 1559 "$BLOCKED_ACCOUNT" 1560 echo -n "Checking that the stale claim was reported... " 1561 check_report_any \ 1562 "row-inconsistency" \ 1563 "row_table" "aggregation_deferrals" 1564 1565 full_reload 1566 cleanup 1567 } 1568 1569 1570 # The other thing an exchange can lawfully be sitting on: an aggregate that 1571 # would not survive its own wire fee. That is not an irregularity and must not 1572 # read as one, so it gets a balance of its own rather than being lumped in with 1573 # payouts nobody has explained. 1574 # 1575 # The reference database has no such aggregate -- every payout in it is worth 1576 # making -- so the test restates the reason on the one deferral it does have. 1577 # Reason 1 is TALER_EXCHANGEDB_DR_AMOUNT_TOO_SMALL, 2 is ..._DR_KYC. Note that 1578 # this leaves the exchange's `aggregation_transient' still naming a 1579 # legitimization requirement, so the -i cross-check must notice that the two 1580 # no longer agree. 1581 function test_16() { 1582 1583 echo "===========16: an aggregate too small to pay out===========" 1584 echo -n "Modifying database: " 1585 # Only the transfer that is still outstanding; the cleared merchant's 1586 # deferral is on file too, but it was overtaken by its wire_out row. 1587 echo "UPDATE exchange.aggregation_deferrals d 1588 SET deferral_reason=1 1589 ,legitimization_requirement_serial_id=0 1590 WHERE NOT EXISTS ( 1591 SELECT 1 FROM exchange.wire_out w 1592 WHERE w.wtid_raw=d.wtid_raw);" \ 1593 | psql -Aqt "$DB" 1594 echo "DONE" 1595 1596 run_audit 1597 check_auditor_running 1598 1599 echo -n "Checking that the funds are booked as a small aggregate... " 1600 check_balance \ 1601 "total_small_aggregate" \ 1602 "$WITHHELD" \ 1603 "Aggregate below the wire fee was not booked as such" 1604 echo -n "Checking that they are no longer an AML hold... " 1605 check_balance \ 1606 "total_aml_hold" \ 1607 "TESTKUDOS:0" \ 1608 "Aggregate below the wire fee still counted as withheld for KYC" 1609 echo -n "Checking that they are not an unexplained lag either... " 1610 check_balance \ 1611 "total_transfer_lag" \ 1612 "TESTKUDOS:0" \ 1613 "Aggregate below the wire fee read as an unexplained delay" 1614 echo -n "Checking that the hold names the new reason... " 1615 check_report "aml-holds" "deferral_reason" "AMOUNT_TOO_SMALL" 1616 echo -n "Checking that the contradiction with the transient was reported... " 1617 check_report_any \ 1618 "row-inconsistency" \ 1619 "row_table" "aggregation_transient" 1620 1621 full_reload 1622 cleanup 1623 } 1624 1625 1626 # *************** Main test loop starts here ************** 1627 1628 1629 # Run all the tests against the database given in $1. 1630 # Sets $fail to 0 on success, non-zero on failure. 1631 function check_with_database() 1632 { 1633 BASEDB="$1" 1634 CONF="$1.conf" 1635 export CONF 1636 echo "Running test suite with database $BASEDB using configuration $CONF" 1637 MASTER_PRIV_FILE="${BASEDB}.mpriv" 1638 taler-exchange-config \ 1639 -f \ 1640 -c "${CONF}" \ 1641 -s exchange-offline \ 1642 -o MASTER_PRIV_FILE \ 1643 -V "${MASTER_PRIV_FILE}" 1644 1645 # Load database 1646 full_reload 1647 1648 # Run test suite 1649 fail=0 1650 for i in $TESTS 1651 do 1652 "test_$i" 1653 if test 0 != $fail 1654 then 1655 break 1656 fi 1657 done 1658 echo "Cleanup (disabled, leaving database $DB behind)" 1659 # dropdb $DB 1660 } 1661 1662 # When the script is not run as root, setup a temporary directory for the 1663 # postgres database. 1664 # Sets PGHOST accordingly to the freshly created socket. 1665 function perform_initdb() { 1666 # Available directly in path? 1667 INITDB_BIN=$(command -v initdb) || true 1668 if [[ -n "$INITDB_BIN" ]]; then 1669 echo " FOUND (in path) at $INITDB_BIN" 1670 else 1671 HAVE_INITDB=$(find /usr -name "initdb" 2> /dev/null \ 1672 | head -1 2> /dev/null \ 1673 | grep postgres) \ 1674 || exit_skip " MISSING" 1675 echo " FOUND at $(dirname "$HAVE_INITDB")" 1676 INITDB_BIN=$(echo "$HAVE_INITDB" | grep bin/initdb | grep postgres | sort -n | tail -n1) 1677 fi 1678 POSTGRES_PATH=$(dirname "$INITDB_BIN") 1679 1680 TMPDIR="$MY_TMP_DIR/postgres" 1681 mkdir -p "$TMPDIR" 1682 echo -n "Setting up Postgres DB at $TMPDIR ..." 1683 $INITDB_BIN \ 1684 --no-sync \ 1685 --auth=trust \ 1686 -D "${TMPDIR}" \ 1687 > "${MY_TMP_DIR}/postgres-dbinit.log" \ 1688 2> "${MY_TMP_DIR}/postgres-dbinit.err" \ 1689 || { 1690 echo "FAILED!" 1691 echo "Last entries in ${MY_TMP_DIR}/postgres-dbinit.err:" 1692 tail "${MY_TMP_DIR}/postgres-dbinit.err" 1693 exit 1 1694 } 1695 echo "DONE" 1696 1697 # Once we move to PG16, we can use: 1698 # --set listen_addresses='' \ 1699 # --set fsync=off \ 1700 # --set max_wal_senders=0 \ 1701 # --set synchronous_commit=off \ 1702 # --set wal_level=minimal \ 1703 # --set unix_socket_directories="${TMPDIR}/sockets" \ 1704 1705 1706 SOCKETDIR="${TMPDIR}/sockets" 1707 mkdir "${SOCKETDIR}" 1708 1709 echo -n "Launching Postgres service" 1710 1711 cat - >> "$TMPDIR/postgresql.conf" <<EOF 1712 unix_socket_directories='${TMPDIR}/sockets' 1713 fsync=off 1714 max_wal_senders=0 1715 synchronous_commit=off 1716 wal_level=minimal 1717 listen_addresses='' 1718 EOF 1719 1720 grep -v host \ 1721 < "$TMPDIR/pg_hba.conf" \ 1722 > "$TMPDIR/pg_hba.conf.new" 1723 mv "$TMPDIR/pg_hba.conf.new" "$TMPDIR/pg_hba.conf" 1724 "${POSTGRES_PATH}/pg_ctl" \ 1725 -D "$TMPDIR" \ 1726 -l "${MY_TMP_DIR}/postgres.log" \ 1727 start \ 1728 > "${MY_TMP_DIR}/postgres-start.log" \ 1729 2> "${MY_TMP_DIR}/postgres-start.err" 1730 echo " DONE" 1731 PGHOST="$TMPDIR/sockets" 1732 export PGHOST 1733 } 1734 1735 1736 # *************** Main logic starts here ************** 1737 1738 # ####### Setup globals ###### 1739 # Postgres database to use (must match configuration file) 1740 export DB="auditor-basedb" 1741 1742 # test required commands exist 1743 echo "Testing for jq" 1744 jq -h > /dev/null || exit_skip "jq required" 1745 echo "Testing for taler-merchant-config" 1746 taler-merchant-config -h > /dev/null || exit_skip "taler-merchant-config required" 1747 echo "Testing for taler-merchant-httpd" 1748 taler-merchant-httpd -h > /dev/null || exit_skip "taler-merchant-httpd required" 1749 echo "Testing for faketime" 1750 faketime -h > /dev/null || exit_skip "faketime required" 1751 # NOTE: really check for all three libeufin commands? 1752 echo "Testing for libeufin" 1753 libeufin-bank --help >/dev/null 2> /dev/null </dev/null || exit_skip "libeufin required" 1754 echo "Testing for taler-wallet-cli" 1755 taler-wallet-cli -h >/dev/null </dev/null 2>/dev/null || exit_skip "taler-wallet-cli required" 1756 1757 1758 echo -n "Testing for Postgres" 1759 1760 MY_TMP_DIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX) 1761 echo "Using $MY_TMP_DIR for logging and temporary data" 1762 1763 # If run as root, simply use the running postgres instance. 1764 # Otherwise create a temporary storage space for postgres. 1765 [ $(id -u) == 0 ] || perform_initdb 1766 1767 MYDIR="${MY_TMP_DIR}/basedb" 1768 mkdir -p "${MYDIR}" 1769 1770 if [ -z ${REUSE_BASEDB_DIR+x} ] 1771 then 1772 echo "Generating fresh database at $MYDIR" 1773 1774 if faketime -f '-1 d' ./generate-kyc-basedb.sh -d "$MYDIR/$DB" 1775 then 1776 echo -n "Reset 'auditor-basedb' database at ${PGHOST:-} ..." 1777 dropdb --if-exists "auditor-basedb" > /dev/null 2> /dev/null || true 1778 createdb "auditor-basedb" || exit_skip "Could not create database '$BASEDB' at ${PGHOST:-}" 1779 echo " DONE" 1780 else 1781 echo "Generation failed" 1782 exit 1 1783 fi 1784 echo "To reuse this database in the future, use:" 1785 echo "export REUSE_BASEDB_DIR=$MY_TMP_DIR" 1786 else 1787 echo "Reusing existing database from ${REUSE_BASEDB_DIR}" 1788 cp -r "${REUSE_BASEDB_DIR}/basedb"/* "${MYDIR}/" 1789 fi 1790 1791 check_with_database "$MYDIR/$DB" 1792 if [ "$fail" != "0" ] 1793 then 1794 exit "$fail" 1795 fi 1796 1797 if [ -z "${REUSE_BASEDB_DIR+x}" ] 1798 then 1799 echo "Run 'export REUSE_BASEDB_DIR=${MY_TMP_DIR}' to re-run tests against the same database" 1800 fi 1801 1802 exit 0