test-kyc.sh (64517B)
1 #!/bin/bash 2 # 3 # This file is part of TALER 4 # Copyright (C) 2014-2026 Taler Systems SA 5 # 6 # TALER is free software; you can redistribute it and/or modify it under the 7 # terms of the GNU General Public License as published by the Free Software 8 # Foundation; either version 3, or (at your option) any later version. 9 # 10 # TALER is distributed in the hope that it will be useful, but WITHOUT ANY 11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 12 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 13 # 14 # You should have received a copy of the GNU General Public License along with 15 # TALER; see the file COPYING. If not, If not, see <http://www.gnu.org/license> 16 # 17 # 18 # shellcheck disable=SC2317 19 # shellcheck disable=SC1091 20 # 21 # 22 # Setup database which was generated from a perfectly normal 23 # exchange-wallet-merchant interaction on a KYC-enabled exchange and run 24 # the auditor against it. 25 # 26 # The reference database (see generate-kyc-basedb.sh) contains two merchant 27 # instances that were paid the same amount and both proved that they own 28 # their bank account with a KYCAUTH wire transfer, but only one of them 29 # then passed the KYC form. The exchange therefore wired the funds of the 30 # first one out and is still holding the funds of the second one. 31 # 32 # The auditor has almost no KYC-specific logic of its own -- see issues.txt 33 # for what it could check and does not -- so this test has two jobs: 34 # 35 # 1. the KYC artefacts (KYCAUTH credits, legitimization measures and 36 # outcomes, an aggregation the exchange lawfully withheld) must not 37 # make the auditor invent findings, and the KYCAUTH totals it does 38 # track -- total_kycauth_in from the bank, total_kycauth_revenue from 39 # the exchange -- must be exactly right; 40 # 2. fault detection must still work on a KYC-enabled exchange, so the 41 # later tests inject the same kinds of faults test-auditor.sh does and 42 # require the auditor to flag them. Tests 2, 8 and 9 pull the two 43 # KYCAUTH totals apart in each of the ways they can disagree, which is 44 # what the reconciliation between them exists to catch. 45 # 46 # Requires 'jq' tool and Postgres superuser rights! 47 set -eu 48 #set -x 49 50 # Set of numbers for all the testcases. 51 # When adding new tests, increase the last number: 52 ALL_TESTS=$(seq 0 16) 53 54 # $TESTS determines which tests we should run. 55 # This construction is used to make it easy to 56 # only run a subset of the tests. To only run a subset, 57 # pass the numbers of the tests to run as the FIRST 58 # argument to test-kyc.sh, i.e.: 59 # 60 # $ test-kyc.sh "1 3" 61 # 62 # to run tests 1 and 3 only. By default, all tests are run. 63 # 64 TESTS=${1:-$ALL_TESTS} 65 66 export TALER_AUDITOR_TOKEN="secret-token:D4CST1Z6AHN3RT03M0T9NSTF2QGHTB5ZD2D3RYZB4HAWG8SX0JEFWBXCKXZHMB7Y3Z7KVFW0B3XPXD5BHCFP8EB0R6CNH2KAWDWVET0" 67 # Make taler-auditor-httpd require the token the requests below send. 68 export TALER_AUDITOR_ACCESS_TOKEN="${TALER_AUDITOR_TOKEN}" 69 export TALER_AUDITOR_SALT="64S36D1N6RVKGC9J6CT3ADHQ70RK4CSM6MV3EE1H68SK8D9P6WW32CHK6GTKCDSR64S36D1N6RVKGC9J6CT3ADHQ70RK4CSM6MV3EE0" 70 71 # Global variable to run the auditor processes under valgrind 72 # VALGRIND=valgrind 73 VALGRIND="" 74 75 . setup.sh 76 77 78 # Cleanup exchange and libeufin between runs. 79 function cleanup() 80 { 81 if [ -n "${EPID:-}" ] 82 then 83 echo -n "Stopping exchange $EPID..." 84 kill -TERM "$EPID" 85 wait "$EPID" || true 86 echo "DONE" 87 unset EPID 88 fi 89 stop_libeufin &> /dev/null 90 } 91 92 # Cleanup to run whenever we exit 93 function exit_cleanup() 94 { 95 jobs 96 if [ -n "${POSTGRES_PATH:-}" ] 97 then 98 echo -n "Stopping Postgres at ${POSTGRES_PATH} ..." 99 "${POSTGRES_PATH}/pg_ctl" \ 100 -D "$TMPDIR" \ 101 --log="${MY_TMP_DIR}/pg_ctl.log" \ 102 stop \ 103 &> ${MY_TMP_DIR}/pg_ctl.out \ 104 || true 105 echo "DONE" 106 fi 107 echo -n "Running exit-cleanup ..." 108 cleanup 109 for n in $(jobs -p) 110 do 111 kill "$n" 2> /dev/null || true 112 done 113 wait || true 114 echo "DONE" 115 } 116 117 # Install cleanup handler (except for kill -9) 118 trap exit_cleanup EXIT 119 120 121 function await_bank () { 122 for n in $(seq 1 80) 123 do 124 echo -n "." 125 sleep 0.1 126 OK=1 127 wget http://localhost:8082/ \ 128 -o /dev/null \ 129 -O /dev/null \ 130 >/dev/null \ 131 && break 132 OK=0 133 done 134 if [ 1 != "$OK" ] 135 then 136 exit_skip "Failed to launch libeufin-bank" 137 fi 138 } 139 140 # Operations to run before the actual audit 141 function pre_audit () { 142 # Launch bank 143 echo -n "Launching libeufin-bank" 144 export CONF 145 export MY_TMP_DIR 146 launch_libeufin 147 await_bank 148 echo " DONE" 149 150 if [ "${1:-no}" = "aggregator" ] 151 then 152 # NO '-y' here: that is --kyc-off, which makes the aggregator 153 # wire funds out without consulting KYC at all and would defeat 154 # the point of this test. 155 echo -n "Running exchange aggregator ..." 156 taler-exchange-aggregator \ 157 -L "INFO" \ 158 -t \ 159 -c "$CONF" \ 160 2> "${MY_TMP_DIR}/aggregator.log" \ 161 || exit_fail "FAIL" 162 echo " DONE" 163 echo -n "Running exchange closer ..." 164 taler-exchange-closer \ 165 -L "INFO" \ 166 -t \ 167 -c "$CONF" \ 168 2> "${MY_TMP_DIR}/closer.log" \ 169 || exit_fail "FAIL" 170 echo " DONE" 171 echo -n "Running exchange transfer ..." 172 taler-exchange-transfer \ 173 -L "INFO" \ 174 -t \ 175 -c "$CONF" \ 176 2> "${MY_TMP_DIR}/transfer.log" \ 177 || exit_fail "FAIL" 178 echo " DONE" 179 fi 180 } 181 182 # actual audit run 183 function audit_only () { 184 # Run the auditor! 185 echo -n "Running audit(s) ..." 186 187 # Restart so that first run is always fresh, and second one is incremental 188 taler-auditor-dbinit \ 189 -r \ 190 -c "$CONF" 191 $VALGRIND taler-helper-auditor-aml \ 192 -i \ 193 -L DEBUG \ 194 -c "$CONF" \ 195 -t \ 196 > "${MY_TMP_DIR}/test-audit-aml.out" \ 197 2> "${MY_TMP_DIR}/test-audit-aml.err" \ 198 || exit_fail "aml audit failed (see ${MY_TMP_DIR}/test-audit-aml.*)" 199 echo -n "." 200 $VALGRIND taler-helper-auditor-aml \ 201 -i \ 202 -L DEBUG \ 203 -c "$CONF" \ 204 -t \ 205 > "${MY_TMP_DIR}/test-audit-aml-inc.out" \ 206 2> "${MY_TMP_DIR}/test-audit-aml-inc.err" \ 207 || exit_fail "incremental aml audit failed (see ${MY_TMP_DIR}/test-audit-aml-inc.*)" 208 echo -n "." 209 $VALGRIND taler-helper-auditor-aggregation \ 210 -L DEBUG \ 211 -c "$CONF" \ 212 -t \ 213 > "${MY_TMP_DIR}/test-audit-aggregation.out" \ 214 2> "${MY_TMP_DIR}/test-audit-aggregation.err" \ 215 || exit_fail "aggregation audit failed (see ${MY_TMP_DIR}/test-audit-aggregation.*)" 216 echo -n "." 217 $VALGRIND taler-helper-auditor-aggregation \ 218 -L DEBUG \ 219 -c "$CONF" \ 220 -t \ 221 > "${MY_TMP_DIR}/test-audit-aggregation-inc.out" \ 222 2> "${MY_TMP_DIR}/test-audit-aggregation-inc.err" \ 223 || exit_fail "incremental aggregation audit failed (see ${MY_TMP_DIR}/test-audit-aggregation-inc.*)" 224 echo -n "." 225 $VALGRIND taler-helper-auditor-coins \ 226 -L DEBUG \ 227 -c "$CONF" \ 228 -t \ 229 > "${MY_TMP_DIR}/test-audit-coins.out" \ 230 2> "${MY_TMP_DIR}/test-audit-coins.err" \ 231 || exit_fail "coin audit failed (see ${MY_TMP_DIR}/test-audit-coins.*)" 232 echo -n "." 233 $VALGRIND taler-helper-auditor-coins \ 234 -L DEBUG \ 235 -c "$CONF" \ 236 -t \ 237 > "${MY_TMP_DIR}/test-audit-coins-inc.out" \ 238 2> "${MY_TMP_DIR}/test-audit-coins-inc.err" \ 239 || exit_fail "incremental coin audit failed (see ${MY_TMP_DIR}/test-audit-coins-inc.*)" 240 echo -n "." 241 $VALGRIND taler-helper-auditor-deposits \ 242 -L DEBUG \ 243 -c "$CONF" \ 244 -t \ 245 > "${MY_TMP_DIR}/test-audit-deposits.out" \ 246 2> "${MY_TMP_DIR}/test-audit-deposits.err" \ 247 || exit_fail "deposits audit failed (see ${MY_TMP_DIR}/test-audit-deposits.*)" 248 echo -n "." 249 $VALGRIND taler-helper-auditor-deposits \ 250 -L DEBUG \ 251 -c "$CONF" \ 252 -t \ 253 > "${MY_TMP_DIR}/test-audit-deposits-inc.out" \ 254 2> "${MY_TMP_DIR}/test-audit-deposits-inc.err" \ 255 || exit_fail "incremental deposits audit failed (see ${MY_TMP_DIR}/test-audit-deposits-inc.*)" 256 echo -n "." 257 $VALGRIND taler-helper-auditor-reserves \ 258 -i \ 259 -L DEBUG \ 260 -c "$CONF" \ 261 -t \ 262 > "${MY_TMP_DIR}/test-audit-reserves.out" \ 263 2> "${MY_TMP_DIR}/test-audit-reserves.err" \ 264 || exit_fail "reserves audit failed (see ${MY_TMP_DIR}/test-audit-reserves.*)" 265 echo -n "." 266 $VALGRIND taler-helper-auditor-reserves \ 267 -i \ 268 -L DEBUG \ 269 -c "$CONF" \ 270 -t \ 271 > "${MY_TMP_DIR}/test-audit-reserves-inc.out" \ 272 2> "${MY_TMP_DIR}/test-audit-reserves-inc.err" \ 273 || exit_fail "incremental reserves audit failed (see ${MY_TMP_DIR}/test-audit-reserves-inc.*)" 274 echo -n "." 275 $VALGRIND taler-helper-auditor-wire-credit \ 276 -i \ 277 -L DEBUG \ 278 -c "$CONF" \ 279 -t \ 280 > "${MY_TMP_DIR}/test-audit-wire-credit.out" \ 281 2> "${MY_TMP_DIR}/test-audit-wire-credit.err" \ 282 || exit_fail "wire credit audit failed (see ${MY_TMP_DIR}/test-audit-wire-credit.*)" 283 echo -n "." 284 $VALGRIND taler-helper-auditor-wire-credit \ 285 -i \ 286 -L DEBUG \ 287 -c "$CONF" \ 288 -t \ 289 > "${MY_TMP_DIR}/test-audit-wire-credit-inc.out" \ 290 2> "${MY_TMP_DIR}/test-audit-wire-credit-inc.err" \ 291 || exit_fail "wire credit audit inc failed (see ${MY_TMP_DIR}/test-audit-wire-credit-inc.*)" 292 echo -n "." 293 $VALGRIND taler-helper-auditor-wire-debit \ 294 -i \ 295 -L DEBUG \ 296 -c "$CONF" \ 297 -t \ 298 > "${MY_TMP_DIR}/test-audit-wire-debit.out" \ 299 2> "${MY_TMP_DIR}/test-audit-wire-debit.err" \ 300 || exit_fail "wire debit audit failed (see ${MY_TMP_DIR}/test-audit-wire-debit.*)" 301 echo -n "." 302 $VALGRIND taler-helper-auditor-wire-debit \ 303 -i \ 304 -L DEBUG \ 305 -c "$CONF" \ 306 -t \ 307 > "${MY_TMP_DIR}/test-audit-wire-debit-inc.out" \ 308 2> "${MY_TMP_DIR}/test-audit-wire-debit-inc.err" \ 309 || exit_fail "wire debit audit inc failed (see ${MY_TMP_DIR}/test-audit-wire-debit-inc.*)" 310 echo -n "." 311 $VALGRIND taler-helper-auditor-purses \ 312 -i \ 313 -L DEBUG \ 314 -c "$CONF" \ 315 -t \ 316 > "${MY_TMP_DIR}/test-audit-purses.out" \ 317 2> "${MY_TMP_DIR}/test-audit-purses.err" \ 318 || exit_fail "audit purses failed" 319 echo -n "." 320 $VALGRIND taler-helper-auditor-purses \ 321 -i \ 322 -L DEBUG \ 323 -c "$CONF" \ 324 -t \ 325 > "${MY_TMP_DIR}/test-audit-purses-inc.out" \ 326 2> "${MY_TMP_DIR}/test-audit-purses-inc.err" \ 327 || exit_fail "audit purses inc failed" 328 echo -n "." 329 $VALGRIND taler-helper-auditor-transfer \ 330 -i \ 331 -L DEBUG \ 332 -c "$CONF" \ 333 -t \ 334 > "${MY_TMP_DIR}/test-audit-transfer.out" \ 335 2> "${MY_TMP_DIR}/test-audit-transfer.err" \ 336 || exit_fail "audit transfer failed" 337 echo -n "." 338 $VALGRIND taler-helper-auditor-transfer \ 339 -i \ 340 -L DEBUG \ 341 -c "$CONF" \ 342 -t \ 343 > "${MY_TMP_DIR}/test-audit-transfer-inc.out" \ 344 2> "${MY_TMP_DIR}/test-audit-transfer-inc.err" \ 345 || exit_fail "audit transfer inc failed" 346 echo -n "." 347 348 echo " DONE" 349 } 350 351 352 # Cleanup to run after the auditor 353 function post_audit () { 354 taler-exchange-dbinit \ 355 -c "$CONF" \ 356 -g \ 357 || exit_fail "exchange DB GC failed" 358 cleanup 359 } 360 361 362 # Run audit process on current database, including report 363 # generation. Pass "aggregator" as $1 to run 364 # $ taler-exchange-aggregator 365 # before auditor (to trigger pending wire transfers). 366 # Pass "drain" as $2 to run a drain operation as well. 367 function run_audit () { 368 pre_audit "${1:-no}" 369 audit_only 370 post_audit 371 } 372 373 374 function stop_auditor_httpd() { 375 if [ -n "${APID:-}" ] 376 then 377 echo -n "Stopping auditor $APID..." 378 kill -TERM "$APID" 379 wait "$APID" || true 380 echo "DONE" 381 unset APID 382 fi 383 } 384 385 386 # Do a full reload of the (original) database 387 function full_reload() 388 { 389 echo -n "Doing full reload of the database (loading ${BASEDB}.sql into $DB at ${PGHOST:-})... " 390 dropdb -f "$DB" &>> ${MY_TMP_DIR}/drop.log || true 391 createdb -T template0 "$DB" \ 392 || exit_skip "could not create database $DB (at ${PGHOST:-})" 393 # Import pre-generated database, -q(ietly) using single (-1) transaction 394 psql -Aqt "$DB" \ 395 -q \ 396 -1 \ 397 -f "${BASEDB}.sql" \ 398 &>> ${MY_TMP_DIR}/postgresql-reload.log \ 399 || exit_skip "Failed to load database $DB from ${BASEDB}.sql" 400 echo "DONE" 401 # Technically, this call shouldn't be needed as libeufin should already be stopped here... 402 stop_libeufin 403 stop_auditor_httpd 404 } 405 406 function run_auditor_httpd() { 407 echo -n "Starting auditor..." 408 $VALGRIND taler-auditor-httpd \ 409 -c "${CONF}" \ 410 -L INFO \ 411 2> "${MY_TMP_DIR}/auditor-httpd.err" & 412 APID=$! 413 414 # Wait for auditor service to be available 415 for n in $(seq 1 50) 416 do 417 echo -n "." 418 sleep 0.2 419 OK=0 420 # auditor 421 wget "http://localhost:8083/config" \ 422 -o /dev/null \ 423 -O /dev/null \ 424 >/dev/null \ 425 || continue 426 OK=1 427 break 428 done 429 echo "... DONE." 430 wget "http://localhost:8083/webui/" \ 431 -o /dev/null \ 432 -O /dev/null \ 433 >/dev/null \ 434 || exit_fail "Auditor WebUI is not available under /webui/" 435 } 436 437 438 function check_auditor_running() { 439 ARUNSTATUS=$(curl -Is http://localhost:8083/config | head -1) 440 if [ -n "${ARUNSTATUS:-}" ] 441 then 442 echo "Auditor running" 443 else 444 echo "Auditor not running, starting it" 445 run_auditor_httpd 446 fi 447 unset ARUNSTATUS 448 } 449 450 function call_endpoint() { 451 if [ -n "${2+x}" ] 452 then 453 curl -s -H "Accept: application/json" -H "Authorization: Bearer ${TALER_AUDITOR_TOKEN}" -o "${MY_TMP_DIR}/${2}.json" "localhost:8083/monitoring/${1}?limit=50&balance_key=${2}" 454 echo "endpoint ${1} called (with balance_key)... " 455 else 456 curl -s -H "Accept: application/json" -H "Authorization: Bearer ${TALER_AUDITOR_TOKEN}" -o "${MY_TMP_DIR}/${1}.json" "localhost:8083/monitoring/${1}?limit=50" 457 echo "endpoint ${1} called... " 458 fi 459 } 460 461 462 function check_balance() { 463 call_endpoint "balances" "$1" 464 BAL=$(jq -r .records[0].balance_value < "${MY_TMP_DIR}/${1}.json") 465 if [ "$BAL" != "$2" ] 466 then 467 exit_fail "$3 (got $BAL, wanted $2)" 468 fi 469 echo "PASS" 470 } 471 472 473 function check_not_balance() { 474 call_endpoint "balances" "$1" 475 BAL=$(jq -r .records[0].balance_value < "${MY_TMP_DIR}/${1}.json") 476 if [ "$BAL" = "$2" ] 477 then 478 exit_fail "$3 (got $BAL, wanted NOT $2)" 479 fi 480 echo "PASS" 481 } 482 483 484 function check_report() { 485 call_endpoint "$1" 486 VAL=$(jq -r .records[0].\"$2\" < "${MY_TMP_DIR}/${1}.json") 487 if [ "$VAL" != "$3" ] 488 then 489 exit_fail "$1::$2 (got $VAL, wanted $3)" 490 fi 491 echo "PASS" 492 } 493 494 # Check that at least one entry of report $1 has field $2 set to $3. 495 # Unlike check_report, this does not depend on the order in which the 496 # auditor inserted its findings: one fault can legitimately be reported 497 # against several operations (a coin with a bad denomination signature is 498 # flagged once per operation that used it). 499 function check_report_any() { 500 call_endpoint "$1" 501 jq -e --arg want "$3" "any(.records[]; .\"$2\" == \$want)" \ 502 < "${MY_TMP_DIR}/${1}.json" \ 503 > /dev/null \ 504 || exit_fail "$1::$2 (no entry with value $3)" 505 echo "PASS" 506 } 507 508 # Check that report $1 has at least one entry, without pinning down any of 509 # its fields. Useful when the injected fault is certain to be found but 510 # the exact amounts depend on how much of a reserve happened to be left. 511 function check_some_report() { 512 call_endpoint "$1" 513 jq -e .records[0] \ 514 < "${MY_TMP_DIR}/${1}.json" \ 515 > /dev/null \ 516 || exit_fail "Wanted a report for $1, but got none" 517 echo "PASS" 518 } 519 520 function check_no_report() { 521 call_endpoint "$1" 522 jq -e .records[0] \ 523 < "${MY_TMP_DIR}/${1}.json" \ 524 > /dev/null \ 525 && exit_fail "Wanted empty report for $1, but got incidents" 526 echo "PASS" 527 } 528 529 function check_report_neg() { 530 call_endpoint "$1" 531 VAL=$(jq -r .records[0].\"$2\" < "${MY_TMP_DIR}/${1}.json") 532 if [ "$VAL" == "$3" ] 533 then 534 exit_fail "$1::$2 (got $VAL, wanted $3)" 535 fi 536 echo "PASS" 537 } 538 539 function check_row() { 540 call_endpoint "$1" 541 if [ -n "${3+x}" ] 542 then 543 RID="$2" 544 WANT="$3" 545 else 546 RID="row_id" 547 WANT="$2" 548 fi 549 ROW=$(jq -r .records[0].\"${RID}\" < "${MY_TMP_DIR}/${1}.json") 550 if [ "$ROW" != "$WANT" ] 551 then 552 exit_fail "Row ${1} wrong (got ${ROW}, wanted ${WANT})" 553 fi 554 echo "PASS" 555 } 556 557 558 559 # Amount the exchange wired to the merchant that passed KYC. 560 CLEARED_WIRED="TESTKUDOS:8.02" 561 # Amount the exchange aggregated for the merchant that did not pass KYC 562 # and then parked in aggregation_transient instead of wiring it out. 563 WITHHELD="TESTKUDOS:8.03" 564 # The account of that merchant. The IBAN libeufin derives from the account 565 # name is stable across regenerations of the reference database, so this can be 566 # hardcoded -- but the wire transfer identifier cannot: the aggregator draws it 567 # with GNUNET_CRYPTO_random_block(), so tests 14 and 15 read it out of the 568 # exchange's own tables instead of naming it. 569 BLOCKED_ACCOUNT="payto://iban/DE12500105170540732493?receiver-name=Blocked44" 570 # Two KYCAUTH wire transfers of TESTKUDOS:0.1, one per merchant instance. 571 KYCAUTH_IN="TESTKUDOS:0.2" 572 # Everything the exchange's bank account was credited with: TESTKUDOS:56 of 573 # reserve top-ups plus the two KYCAUTH transfers above. The KYCAUTH part is 574 # revenue rather than a liability, but it is money in the account all the 575 # same, so it belongs in the balance the account is expected to have. 576 WIRE_IN="TESTKUDOS:56.2" 577 578 579 # Assert that all of the auditor's loss, delta and irregularity balances 580 # are zero, i.e. that nothing about this database looks wrong to it. 581 function check_all_clean() { 582 echo -n "Test for emergencies... " 583 check_no_report "emergencies" 584 echo -n "Test for emergencies by count... " 585 check_no_report "emergencies-by-count" 586 echo -n "Test for deposit confirmation problems... " 587 check_no_report "deposit-confirmations" 588 echo -n "Test for denomination key validity problems... " 589 check_no_report "denomination-key-validity-withdraw-inconsistencies" 590 echo -n "Test for denominations without signatures... " 591 check_no_report "denominations-without-sigs" 592 echo -n "Test for wire out inconsistencies... " 593 check_no_report "wire-out-inconsistencies" 594 echo -n "Test for reserve in inconsistencies... " 595 check_no_report "reserve-in-inconsistencies" 596 echo -n "Test for KYCAUTH in inconsistencies... " 597 check_no_report "kycauth-in-inconsistencies" 598 echo -n "Test for misattribution inconsistencies... " 599 check_no_report "misattribution-in-inconsistencies" 600 echo -n "Test for row inconsistencies... " 601 check_no_report "row-inconsistencies" 602 echo -n "Test for minor row inconsistencies... " 603 check_no_report "row-minor-inconsistencies" 604 echo -n "Test for wire format inconsistencies... " 605 check_no_report "wire-format-inconsistencies" 606 echo -n "Test for arithmetic inconsistencies... " 607 check_no_report "amount-arithmetic-inconsistencies" 608 echo -n "Test for coin inconsistencies... " 609 check_no_report "coin-inconsistencies" 610 echo -n "Test for bad signature losses... " 611 check_no_report "bad-sig-losses" 612 echo -n "Test for fee time inconsistencies... " 613 check_no_report "fee-time-inconsistencies" 614 echo -n "Test for purses not closed... " 615 check_no_report "purse-not-closed-inconsistencies" 616 echo -n "Test for reserves not closed... " 617 check_no_report "reserve-not-closed-inconsistencies" 618 echo -n "Test for insufficient reserve balances... " 619 check_no_report "reserve-balance-insufficient-inconsistencies" 620 echo -n "Test for wrong reserve balance summaries... " 621 check_no_report "reserve-balance-summary-wrong-inconsistencies" 622 echo -n "Test for closure lags... " 623 check_no_report "closure-lags" 624 625 # Just to test the endpoint and for logging ... 626 call_endpoint "balances" 627 628 echo -n "Testing aggregation bad sig loss balance... " 629 check_balance \ 630 "aggregation_total_bad_sig_loss" \ 631 "TESTKUDOS:0" \ 632 "Wrong total bad sig loss from aggregation" 633 echo -n "Testing coin irregular loss balance... " 634 check_balance \ 635 "coin_irregular_loss" \ 636 "TESTKUDOS:0" \ 637 "Wrong total irregular loss from coins" 638 echo -n "Testing reserves bad sig loss balance... " 639 check_balance \ 640 "reserves_total_bad_sig_loss" \ 641 "TESTKUDOS:0" \ 642 "Wrong total bad sig loss from reserves" 643 echo -n "Testing purse bad sig loss balance... " 644 check_balance \ 645 "purse_total_bad_sig_loss" \ 646 "TESTKUDOS:0" \ 647 "Wrong total bad sig loss from purses" 648 649 echo -n "Test for bad incoming delta plus... " 650 check_balance \ 651 "total_bad_amount_in_plus" \ 652 "TESTKUDOS:0" \ 653 "Expected total wire in delta plus wrong" 654 echo -n "Test for bad incoming delta minus... " 655 check_balance \ 656 "total_bad_amount_in_minus" \ 657 "TESTKUDOS:0" \ 658 "Expected total wire in delta minus wrong" 659 echo -n "Test for bad outgoing delta plus... " 660 check_balance \ 661 "total_bad_amount_out_plus" \ 662 "TESTKUDOS:0" \ 663 "Expected total wire out delta plus wrong" 664 echo -n "Test for bad outgoing delta minus... " 665 check_balance \ 666 "total_bad_amount_out_minus" \ 667 "TESTKUDOS:0" \ 668 "Expected total wire out delta minus wrong" 669 echo -n "Test for misattribution amounts... " 670 check_balance \ 671 "total_misattribution_in" \ 672 "TESTKUDOS:0" \ 673 "Expected total misattribution in wrong" 674 675 echo -n "Test for aggregation wire out delta plus... " 676 check_balance \ 677 "aggregation_total_wire_out_delta_plus" \ 678 "TESTKUDOS:0" \ 679 "Expected total wire out delta plus wrong" 680 echo -n "Test for aggregation wire out delta minus... " 681 check_balance \ 682 "aggregation_total_wire_out_delta_minus" \ 683 "TESTKUDOS:0" \ 684 "Expected total wire out delta minus wrong" 685 686 echo -n "Checking for unexpected aggregation delta plus... " 687 check_balance \ 688 "aggregation_total_arithmetic_delta_plus" \ 689 "TESTKUDOS:0" \ 690 "Wrong arithmetic delta plus from aggregations" 691 echo -n "Checking for unexpected aggregation delta minus... " 692 check_balance \ 693 "aggregation_total_arithmetic_delta_minus" \ 694 "TESTKUDOS:0" \ 695 "Wrong arithmetic delta minus from aggregations" 696 echo -n "Checking for unexpected coin delta plus... " 697 check_balance \ 698 "coins_total_arithmetic_delta_plus" \ 699 "TESTKUDOS:0" \ 700 "Wrong arithmetic delta plus from coins" 701 echo -n "Checking for unexpected coin delta minus... " 702 check_balance \ 703 "coins_total_arithmetic_delta_minus" \ 704 "TESTKUDOS:0" \ 705 "Wrong arithmetic delta minus from coins" 706 echo -n "Checking for unexpected reserves delta plus... " 707 check_balance \ 708 "reserves_total_arithmetic_delta_plus" \ 709 "TESTKUDOS:0" \ 710 "Wrong arithmetic delta plus from reserves" 711 echo -n "Checking for unexpected reserves delta minus... " 712 check_balance \ 713 "reserves_total_arithmetic_delta_minus" \ 714 "TESTKUDOS:0" \ 715 "Wrong arithmetic delta minus from reserves" 716 } 717 718 719 # The KYC-specific assertions: how much arrived by KYCAUTH, how much left 720 # the exchange, and how much it is sitting on. 721 function check_kyc_state() { 722 echo -n "Checking the KYCAUTH credits were accounted for... " 723 check_balance \ 724 "total_kycauth_in" \ 725 "$KYCAUTH_IN" \ 726 "Wrong total for KYCAUTH wire transfers" 727 728 # KYCAUTH money is not credited to a reserve and never paid back, so 729 # the auditor books it as revenue of the exchange operator and counts 730 # it towards the balance the bank account is expected to have. 731 echo -n "Checking the KYCAUTH credits were booked as revenue... " 732 check_balance \ 733 "total_kycauth_revenue" \ 734 "$KYCAUTH_IN" \ 735 "Wrong KYCAUTH revenue" 736 echo -n "Checking the KYCAUTH credits reached the balance sheet... " 737 check_balance \ 738 "total_wire_in" \ 739 "$WIRE_IN" \ 740 "Wrong total credited to the exchange" 741 742 echo -n "Checking that only the cleared merchant was paid... " 743 check_balance \ 744 "total_wire_out" \ 745 "$CLEARED_WIRED" \ 746 "Wrong total wired out" 747 748 # The exchange aggregated the blocked merchant's coins, wrote the 749 # aggregation_tracking rows and then parked the payout in 750 # aggregation_transient rather than executing it. Confirm from the 751 # exchange's own database that the money really is still there, so 752 # that the auditor-side assertion below is about the auditor and not 753 # about the reference database having drifted. 754 echo -n "Checking that the exchange withheld the blocked payout... " 755 TRANSIENT=$(psql -Aqt "$DB" \ 756 -c "SELECT COUNT(*) FROM exchange.aggregation_transient;") 757 if [ "$TRANSIENT" != "1" ] 758 then 759 exit_fail "Expected one withheld aggregation, got ${TRANSIENT}" 760 fi 761 WIRED=$(psql -Aqt "$DB" -c "SELECT COUNT(*) FROM exchange.wire_out;") 762 if [ "$WIRED" != "1" ] 763 then 764 exit_fail "Expected exactly one wire transfer, got ${WIRED}" 765 fi 766 echo "PASS" 767 768 # The exchange also said *why*, in the append-only table that replication 769 # can carry. aggregation_transient above is the same claim, but an 770 # external auditor never gets to see it, so this is the row the auditor's 771 # classification actually rests on. Reason 2 is 772 # TALER_EXCHANGEDB_DR_KYC. 773 # 774 # Restrict this to transfers that have not gone out. Both merchants were 775 # withheld on the generator's first aggregator run -- neither had passed 776 # KYC yet -- so the cleared one has a deferral of its own, made obsolete by 777 # the wire_out row that followed. Append-only means those stay on file, 778 # which is the point: the record of a hold outlives the hold. 779 echo -n "Checking that the exchange recorded why it withheld it... " 780 DEFERRED=$(psql -Aqt "$DB" \ 781 -c "SELECT COUNT(*) FROM exchange.aggregation_deferrals d 782 WHERE d.deferral_reason=2 783 AND d.legitimization_requirement_serial_id != 0 784 AND NOT EXISTS ( 785 SELECT 1 FROM exchange.wire_out w 786 WHERE w.wtid_raw=d.wtid_raw);") 787 if [ "$DEFERRED" != "1" ] 788 then 789 exit_fail "Expected one open KYC deferral on record, got ${DEFERRED}" 790 fi 791 echo "PASS" 792 793 # ... and the auditor sees it. The withheld deposits do carry 794 # aggregation_tracking rows, so they are not "lagging" in the sense of 795 # total_amount_lag, which counts deposits the exchange has not aggregated 796 # at all; that balance is legitimately zero here. What the exchange has 797 # not done is the wire transfer, and that is what total_aml_hold counts. 798 echo -n "Checking that nothing is waiting to be aggregated... " 799 check_balance \ 800 "total_amount_lag" \ 801 "TESTKUDOS:0" \ 802 "Wrong total for deposits that were never aggregated" 803 echo -n "Checking that the withheld payout is booked as an AML hold... " 804 check_balance \ 805 "total_aml_hold" \ 806 "$WITHHELD" \ 807 "Wrong total withheld pending legitimization" 808 # The blocked merchant is the *only* transfer the exchange owes, and its 809 # KYC measure is open, so nothing may land in the unexplained bucket. 810 # Were the two ever to swap, the exchange would be stalling a payout it 811 # has no legal reason to stall, which is the case this split exists for. 812 echo -n "Checking that no payout is unexplained... " 813 check_balance \ 814 "total_transfer_lag" \ 815 "TESTKUDOS:0" \ 816 "Exchange is withholding funds it gave no reason for" 817 # Every deposit here is aggregated into a payout worth making, so the 818 # third bucket -- money parked only because the aggregate would not 819 # cover its wire fee -- must be empty. It exists so that such money 820 # does not read as either of the two above, both of which say something 821 # is wrong. 822 echo -n "Checking that nothing is parked as a small aggregate... " 823 check_balance \ 824 "total_small_aggregate" \ 825 "TESTKUDOS:0" \ 826 "Exchange parked an aggregate as too small to pay out" 827 echo -n "Checking that the hold names the blocked account... " 828 check_report_any \ 829 "aml-holds" \ 830 "account" \ 831 "$BLOCKED_ACCOUNT" 832 echo -n "Checking that the hold names the amount... " 833 check_report "aml-holds" "amount" "$WITHHELD" 834 echo -n "Checking that the hold is attributed to KYC... " 835 check_report "aml-holds" "deferral_reason" "KYC" 836 echo -n "Test for pending deposits... " 837 check_no_report "pending-deposits" 838 echo -n "Test for early aggregations... " 839 check_no_report "early-aggregations" 840 } 841 842 843 # The KYC artefacts must not make the auditor invent findings. 844 function test_0() { 845 846 echo "===========0: normal run with aggregator===========" 847 run_audit aggregator 848 check_auditor_running 849 850 echo "Checking output" 851 check_all_clean 852 check_kyc_state 853 854 # cannot easily undo aggregator, hence full reload 855 full_reload 856 cleanup 857 } 858 859 860 # Same, but without letting the aggregator run first. The aggregator 861 # cannot lawfully do anything more than it already did when the reference 862 # database was generated, so the auditor's view must be identical. 863 function test_1() { 864 865 echo "===========1: normal run===========" 866 run_audit 867 check_auditor_running 868 869 echo "Checking output" 870 check_all_clean 871 check_kyc_state 872 873 # Database was unmodified, no need to undo 874 echo "OK" 875 } 876 877 878 # A KYCAUTH credit that the bank does not agree is a KYCAUTH credit. 879 function test_2() { 880 881 echo "===========2: KYCAUTH credit reclassified by the bank===========" 882 # Modifying the bank's books needs libeufin to be restarted, it does 883 # not re-read them. 884 pre_audit 885 stop_libeufin 886 echo -n "Modifying database: " 887 # libeufin records for every credit to the exchange whether it was a 888 # reserve top-up or an account authorisation. Turn one of the two 889 # KYCAUTH credits into a reserve credit: the KYCAUTH total the auditor 890 # tracks must drop by that amount, and because the 32 bytes of KYCAUTH 891 # metadata are not a reserve public key the exchange knows, the credit 892 # now also looks like an incoming transfer the exchange never booked. 893 echo "UPDATE libeufin_bank.taler_exchange_incoming SET type='reserve' WHERE exchange_incoming_id=(SELECT MIN(exchange_incoming_id) FROM libeufin_bank.taler_exchange_incoming WHERE type='kyc')" \ 894 | psql -Aqt "$DB" 895 echo "DONE" 896 launch_libeufin 897 await_bank 898 899 audit_only 900 check_auditor_running 901 902 echo -n "Checking that the KYCAUTH total dropped... " 903 check_balance \ 904 "total_kycauth_in" \ 905 "TESTKUDOS:0.1" \ 906 "Wrong total for KYCAUTH wire transfers" 907 # Only the bank's books were touched, so the exchange still claims the 908 # KYCAUTH revenue for a transfer the bank no longer reports as one. 909 echo -n "Checking that the exchange still claims the revenue... " 910 check_balance \ 911 "total_kycauth_revenue" \ 912 "$KYCAUTH_IN" \ 913 "Wrong KYCAUTH revenue" 914 echo -n "Checking that the unbooked credit was flagged... " 915 check_report_any \ 916 "reserve-in-inconsistencies" \ 917 "diagnostic" "unknown to exchange" 918 echo -n "Checking that it was counted as an excess credit... " 919 check_balance \ 920 "total_bad_amount_in_plus" \ 921 "TESTKUDOS:0.1" \ 922 "Wrong total_bad_amount_in_plus" 923 924 # ... and the other side of the same disagreement: the exchange has a 925 # kycauths_in row for a transfer the bank now says was something else, 926 # so the auditor must not take the exchange's word for it. 927 echo -n "Checking that the unconfirmed KYCAUTH was flagged... " 928 check_report_any \ 929 "kycauth-in-inconsistencies" \ 930 "diagnostic" "KYCAUTH wire transfer claimed by exchange not found" 931 echo -n "Checking the amount the exchange claims... " 932 check_report_any \ 933 "kycauth-in-inconsistencies" \ 934 "amount_exchange_expected" "TESTKUDOS:0.1" 935 echo -n "Checking that it was counted as a missing credit... " 936 check_balance \ 937 "total_bad_amount_in_minus" \ 938 "TESTKUDOS:0.1" \ 939 "Wrong total_bad_amount_in_minus" 940 941 post_audit 942 full_reload 943 cleanup 944 } 945 946 947 # The rest of the tests inject the same kinds of faults test-auditor.sh 948 # injects, to show that KYC does not stop the auditor from finding them. 949 950 951 # Change the amount of an incoming wire transfer as recorded by the 952 # exchange. 953 function test_3() { 954 955 echo "===========3: reserves_in inconsistency===========" 956 echo -n "Modifying database: " 957 echo "UPDATE exchange.reserves_in SET credit.val=5 WHERE reserve_in_serial_id=1" \ 958 | psql -Aqt "$DB" 959 echo "DONE" 960 961 run_audit 962 check_auditor_running 963 964 echo -n "Testing inconsistency detection... " 965 check_report \ 966 "reserve-in-inconsistencies" \ 967 "row_id" 1 968 echo -n "Testing amount wired... " 969 check_report \ 970 "reserve-in-inconsistencies" \ 971 "amount_wired" "TESTKUDOS:10" 972 echo -n "Testing amount expected... " 973 check_report \ 974 "reserve-in-inconsistencies" \ 975 "amount_exchange_expected" "TESTKUDOS:5" 976 echo -n "Checking wire credit balance plus... " 977 check_balance \ 978 "total_bad_amount_in_plus" \ 979 "TESTKUDOS:5" \ 980 "Expected total_bad_amount_in_plus wrong" 981 982 full_reload 983 cleanup 984 } 985 986 987 # Test where denom_sig in the known_coins table is wrong (=> bad signature) 988 function test_4() { 989 990 echo "===========4: known_coins signature wrong===========" 991 OLD_ROW=$(echo "SELECT known_coin_id FROM exchange.known_coins LIMIT 1;" | psql "$DB" -Aqt) 992 COIN_PUB=$(echo "SELECT coin_pub FROM exchange.known_coins WHERE known_coin_id=$OLD_ROW;" | psql "$DB" -Aqt) 993 # shellcheck disable=SC2028 994 echo "UPDATE exchange.known_coins SET denom_sig='\x0000000100000000287369672d76616c200a2028727361200a2020287320233542383731423743393036444643303442424430453039353246413642464132463537303139374131313437353746324632323332394644443146324643333445393939413336363430334233413133324444464239413833353833464536354442374335434445304441453035374438363336434541423834463843323843344446304144363030343430413038353435363039373833434431333239393736423642433437313041324632414132414435413833303432434346314139464635394244434346374436323238344143354544364131373739463430353032323241373838423837363535453434423145443831364244353638303232413123290a2020290a20290b' WHERE coin_pub='$COIN_PUB'" \ 995 | psql -Aqt "$DB" 996 997 run_audit 998 check_auditor_running 999 1000 echo -n "Checking bad-signature-loss detected... " 1001 check_report_neg \ 1002 "bad-sig-losses" \ 1003 "loss" "TESTKUDOS:0" 1004 echo -n "Checking bad-signature-loss balance update... " 1005 check_not_balance \ 1006 "coin_irregular_loss" \ 1007 "TESTKUDOS:0" \ 1008 "Wrong total bad sig loss" 1009 1010 full_reload 1011 cleanup 1012 } 1013 1014 1015 # Test where the amount the bank actually wired differs from what the 1016 # exchange claims. 1017 function test_5() { 1018 1019 echo "===========5: incorrect wire_out amount===========" 1020 pre_audit 1021 stop_libeufin 1022 echo -n "Modifying database: " 1023 echo "UPDATE libeufin_bank.bank_account_transactions SET amount=(50,0) WHERE debtor_name='Exchange Company';" \ 1024 | psql -Aqt "$DB" 1025 echo "DONE" 1026 launch_libeufin 1027 await_bank 1028 1029 audit_only 1030 check_auditor_running 1031 1032 echo -n "Testing wire-out-inconsistency expected... " 1033 check_report \ 1034 "wire-out-inconsistencies" \ 1035 "expected" \ 1036 "$CLEARED_WIRED" 1037 echo -n "Testing wire-out-inconsistency claimed... " 1038 check_report \ 1039 "wire-out-inconsistencies" \ 1040 "claimed" \ 1041 "TESTKUDOS:50" 1042 echo -n "Testing bad_amount_out_plus balance reporting... " 1043 check_not_balance \ 1044 "total_bad_amount_out_plus" \ 1045 "TESTKUDOS:0" \ 1046 "reported total_bad_amount_out_plus wrong" 1047 1048 post_audit 1049 full_reload 1050 cleanup 1051 } 1052 1053 1054 # Test where the exchange claims to have received more than the bank sent, 1055 # which also throws its reserve balance summary off. 1056 function test_6() { 1057 1058 echo "===========6: reserve balance summary wrong===========" 1059 echo -n "Modifying database: " 1060 echo "UPDATE exchange.reserves_in SET credit.val=15 WHERE reserve_in_serial_id=1" \ 1061 | psql -Aqt "$DB" 1062 echo "DONE" 1063 1064 run_audit 1065 check_auditor_running 1066 1067 echo -n "Checking reserve balance summary inconsistency detected... " 1068 check_some_report \ 1069 "reserve-balance-summary-wrong-inconsistencies" 1070 echo -n "Testing amount the exchange expected... " 1071 check_report \ 1072 "reserve-in-inconsistencies" \ 1073 "amount_exchange_expected" "TESTKUDOS:15" 1074 echo -n "Checking wire credit balance minus... " 1075 check_balance \ 1076 "total_bad_amount_in_minus" \ 1077 "TESTKUDOS:5" \ 1078 "Wrong total_bad_amount_in_minus" 1079 1080 full_reload 1081 cleanup 1082 } 1083 1084 1085 # Test where the exchange forgot a deposit that a merchant can prove. 1086 function test_7() { 1087 1088 echo "===========7: deposit wire target malformed===========" 1089 SERIAL=$(echo "SELECT batch_deposit_serial_id FROM exchange.coin_deposits WHERE (amount_with_fee).val=3 ORDER BY batch_deposit_serial_id LIMIT 1" | psql "$DB" -Aqt) 1090 # shellcheck disable=SC2028 1091 echo "INSERT INTO exchange.wire_targets (payto_uri, wire_target_h_payto) VALUES ('payto://x-taler-bank/localhost/testuser-xxlargtp', '\x1e8f31936b3cee8f8afd3aac9e38b5db42d45b721ffc4eb1e5b9ddaf1565660b');" \ 1092 | psql "$DB" -Aqt 1093 # shellcheck disable=SC2028 1094 echo "UPDATE exchange.batch_deposits SET wire_target_h_payto='\x1e8f31936b3cee8f8afd3aac9e38b5db42d45b721ffc4eb1e5b9ddaf1565660b' WHERE batch_deposit_serial_id=${SERIAL};" \ 1095 | psql -Aqt "$DB" 1096 1097 run_audit 1098 check_auditor_running 1099 1100 echo -n "Checking correct operation of loss reported... " 1101 check_report \ 1102 "bad-sig-losses" \ 1103 "operation" "deposit" 1104 echo -n "Checking correct problem row ID reported... " 1105 check_report \ 1106 "bad-sig-losses" \ 1107 "problem_row_id" "$SERIAL" 1108 echo -n "Checking irregular coin loss... " 1109 check_not_balance \ 1110 "coin_irregular_loss" \ 1111 "TESTKUDOS:0" \ 1112 "Wrong total irregular coin loss" 1113 1114 full_reload 1115 cleanup 1116 } 1117 1118 1119 # The reverse of test 2: the bank reports a KYCAUTH transfer that the 1120 # exchange has no record of. An exchange that could make such a row 1121 # disappear could also make the account authorisation it justified 1122 # disappear, so this must be flagged even though the money involved is 1123 # small. 1124 function test_8() { 1125 1126 echo "===========8: KYCAUTH credit missing from the exchange===========" 1127 echo -n "Modifying database: " 1128 echo "DELETE FROM exchange.kycauths_in WHERE kycauth_in_serial_id=1" \ 1129 | psql -Aqt "$DB" 1130 echo "DONE" 1131 1132 run_audit 1133 check_auditor_running 1134 1135 echo -n "Checking that the bank's view is unchanged... " 1136 check_balance \ 1137 "total_kycauth_in" \ 1138 "$KYCAUTH_IN" \ 1139 "Wrong total for KYCAUTH wire transfers" 1140 echo -n "Checking that the exchange claims less revenue... " 1141 check_balance \ 1142 "total_kycauth_revenue" \ 1143 "TESTKUDOS:0.1" \ 1144 "Wrong KYCAUTH revenue" 1145 echo -n "Checking that the unbooked KYCAUTH was flagged... " 1146 check_report_any \ 1147 "kycauth-in-inconsistencies" \ 1148 "diagnostic" "unknown to exchange" 1149 echo -n "Checking the amount the bank reports... " 1150 check_report_any \ 1151 "kycauth-in-inconsistencies" \ 1152 "amount_wired" "TESTKUDOS:0.1" 1153 echo -n "Checking that it was counted as an excess credit... " 1154 check_balance \ 1155 "total_bad_amount_in_plus" \ 1156 "TESTKUDOS:0.1" \ 1157 "Wrong total_bad_amount_in_plus" 1158 1159 full_reload 1160 cleanup 1161 } 1162 1163 1164 # The exchange booked a KYCAUTH transfer, but for a different amount than 1165 # the bank actually moved. 1166 function test_9() { 1167 1168 echo "===========9: KYCAUTH amount inconsistency===========" 1169 echo -n "Modifying database: " 1170 echo "UPDATE exchange.kycauths_in SET credit.val=5 WHERE kycauth_in_serial_id=1" \ 1171 | psql -Aqt "$DB" 1172 echo "DONE" 1173 1174 run_audit 1175 check_auditor_running 1176 1177 echo -n "Checking that the mismatch was flagged... " 1178 check_report_any \ 1179 "kycauth-in-inconsistencies" \ 1180 "diagnostic" "wire amount does not match" 1181 echo -n "Checking the amount the exchange claims... " 1182 check_report_any \ 1183 "kycauth-in-inconsistencies" \ 1184 "amount_exchange_expected" "TESTKUDOS:5.1" 1185 echo -n "Checking the amount the bank reports... " 1186 check_report_any \ 1187 "kycauth-in-inconsistencies" \ 1188 "amount_wired" "TESTKUDOS:0.1" 1189 echo -n "Checking that the shortfall was counted... " 1190 check_balance \ 1191 "total_bad_amount_in_minus" \ 1192 "TESTKUDOS:5" \ 1193 "Wrong total_bad_amount_in_minus" 1194 echo -n "Checking that the inflated revenue reached the balance sheet... " 1195 check_balance \ 1196 "total_kycauth_revenue" \ 1197 "TESTKUDOS:5.2" \ 1198 "Wrong KYCAUTH revenue" 1199 1200 full_reload 1201 cleanup 1202 } 1203 1204 1205 # The auditor may well see a KYCAUTH transfer at the bank before the 1206 # exchange has booked it. It reports that, and must take the report back 1207 # -- together with the excess credit it booked alongside -- once the 1208 # exchange catches up. Without the retraction of the balance, a purely 1209 # transient disagreement would inflate total_bad_amount_in_plus forever. 1210 function test_10() { 1211 1212 echo "===========10: KYCAUTH finding retracted when the exchange catches up===========" 1213 echo -n "Modifying database: " 1214 # Hide the exchange's record of the *last* KYCAUTH transfer, keeping a 1215 # copy so it can be restored byte for byte. It has to be the last one: 1216 # kycauth_in_cb() resumes from a progress point, so a row reinstated 1217 # below that point would never be looked at again. 1218 echo "CREATE TABLE stashed_kycauth AS 1219 SELECT * FROM exchange.kycauths_in 1220 WHERE kycauth_in_serial_id 1221 = (SELECT MAX(kycauth_in_serial_id) FROM exchange.kycauths_in); 1222 DELETE FROM exchange.kycauths_in 1223 WHERE kycauth_in_serial_id 1224 = (SELECT MAX(kycauth_in_serial_id) FROM exchange.kycauths_in);" \ 1225 | psql -Aqt "$DB" 1226 echo "DONE" 1227 1228 pre_audit 1229 audit_only 1230 check_auditor_running 1231 1232 echo -n "Checking that the unbooked transfer was reported... " 1233 check_report_any \ 1234 "kycauth-in-inconsistencies" \ 1235 "diagnostic" "unknown to exchange" 1236 echo -n "Checking that it was counted as an excess credit... " 1237 check_balance \ 1238 "total_bad_amount_in_plus" \ 1239 "TESTKUDOS:0.1" \ 1240 "Wrong total_bad_amount_in_plus" 1241 1242 # The exchange catches up. No progress points are touched: the credit 1243 # audit is simply run once more, exactly as a resident helper would on 1244 # its next wake-up. 1245 echo -n "Letting the exchange catch up: " 1246 echo "INSERT INTO exchange.kycauths_in SELECT * FROM stashed_kycauth; 1247 DROP TABLE stashed_kycauth;" \ 1248 | psql -Aqt "$DB" 1249 echo "DONE" 1250 1251 echo -n "Re-running the credit audit ..." 1252 $VALGRIND taler-helper-auditor-wire-credit \ 1253 -i \ 1254 -L DEBUG \ 1255 -c "$CONF" \ 1256 -t \ 1257 > "${MY_TMP_DIR}/test-audit-wire-credit-retract.out" \ 1258 2> "${MY_TMP_DIR}/test-audit-wire-credit-retract.err" \ 1259 || exit_fail "wire credit re-audit failed (see ${MY_TMP_DIR}/test-audit-wire-credit-retract.*)" 1260 echo " DONE" 1261 1262 echo -n "Checking that the report was retracted... " 1263 check_no_report "kycauth-in-inconsistencies" 1264 echo -n "Checking that the excess credit was retracted too... " 1265 check_balance \ 1266 "total_bad_amount_in_plus" \ 1267 "TESTKUDOS:0" \ 1268 "Retracted finding left total_bad_amount_in_plus inflated" 1269 1270 post_audit 1271 full_reload 1272 cleanup 1273 } 1274 1275 1276 # An AML staff appointment that the exchange's offline master key did not 1277 # sign. Without this check an exchange could appoint an AML officer, or 1278 # quietly widen a read-only officer to read-write, with the offline key never 1279 # having been involved. 1280 function test_11() { 1281 1282 echo "===========11: AML staff appointment with a bad master signature===========" 1283 echo -n "Modifying database: " 1284 # shellcheck disable=SC2028 1285 echo "INSERT INTO exchange.aml_staff 1286 (decider_pub, master_sig, decider_name, is_active, read_only, last_change) 1287 VALUES 1288 ('\x1111111111111111111111111111111111111111111111111111111111111111', 1289 '\x22222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222222', 1290 'Unappointed Officer', TRUE, FALSE, 1000000000000000);" \ 1291 | psql -Aqt "$DB" 1292 echo "DONE" 1293 1294 run_audit 1295 check_auditor_running 1296 1297 echo -n "Checking that the appointment was flagged... " 1298 check_report_any \ 1299 "row-inconsistencies" \ 1300 "diagnostic" "invalid master key signature on staff status" 1301 echo -n "Checking that it was blamed on the right table... " 1302 check_report_any \ 1303 "row-inconsistencies" \ 1304 "row_table" "aml_staff" 1305 1306 # aml_staff is append-only, so the helper can and must resume from where 1307 # it stopped instead of re-reading the whole table on every wake-up. 1308 # Both halves have to hold: the cursor has to be stored at all, and the 1309 # second run has to pick it up. 1310 echo -n "Checking that the aml_staff cursor was stored... " 1311 AML_PP=$(psql -Aqt "$DB" \ 1312 -c "SELECT progress_offset FROM auditor.auditor_progress \ 1313 WHERE progress_key='aml_staff_uuid';") 1314 if [ -z "$AML_PP" ] || [ "$AML_PP" = "0" ] 1315 then 1316 exit_fail "aml_staff_uuid progress point not recorded (got '$AML_PP')" 1317 fi 1318 echo "PASS ($AML_PP)" 1319 echo -n "Checking that the second run resumed from it... " 1320 if ! grep -q "Resuming AML audit at $AML_PP/" \ 1321 "${MY_TMP_DIR}/test-audit-aml-inc.err" 1322 then 1323 exit_fail "incremental AML audit did not resume at $AML_PP" 1324 fi 1325 echo "PASS" 1326 1327 full_reload 1328 cleanup 1329 } 1330 1331 1332 # An AML decision that no valid officer signature backs, by somebody the 1333 # exchange never appointed at all. Both halves have to be reported: the 1334 # signature is what makes the decision accountable, and the appointment is 1335 # what makes the signature mean anything. 1336 function test_12() { 1337 1338 echo "===========12: AML decision by an unappointed officer===========" 1339 echo -n "Modifying database: " 1340 # shellcheck disable=SC2028 1341 echo "INSERT INTO exchange.aml_history 1342 (h_payto, justification, decider_pub, decider_sig, outcome_serial_id) 1343 SELECT h_payto, 1344 'injected by test 12', 1345 '\x3333333333333333333333333333333333333333333333333333333333333333', 1346 '\x44444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444444', 1347 outcome_serial_id 1348 FROM exchange.legitimization_outcomes 1349 ORDER BY outcome_serial_id 1350 LIMIT 1;" \ 1351 | psql -Aqt "$DB" 1352 echo "DONE" 1353 1354 run_audit 1355 check_auditor_running 1356 1357 echo -n "Checking that the bad signature was flagged... " 1358 check_report_any \ 1359 "row-inconsistencies" \ 1360 "diagnostic" "invalid officer signature on decision" 1361 echo -n "Checking that the missing appointment was flagged... " 1362 check_report_any \ 1363 "row-inconsistencies" \ 1364 "diagnostic" "officer was not appointed when the decision was made" 1365 echo -n "Checking that it was blamed on the right table... " 1366 check_report_any \ 1367 "row-inconsistencies" \ 1368 "row_table" "aml_history" 1369 1370 full_reload 1371 cleanup 1372 } 1373 1374 1375 # `legitimization_outcomes' is the table that says which KYC rules apply to 1376 # an account, so a row nothing produced is how an exchange would quietly 1377 # exempt a customer from KYC. The account here is fresh: it has no AML 1378 # decision pointing at the outcome, no legitimization process, and no earlier 1379 # outcome that expired, which is all three of the ways the exchange's own 1380 # code creates one. 1381 function test_13() { 1382 1383 echo "===========13: legitimization outcome without a justification===========" 1384 echo -n "Modifying database: " 1385 # shellcheck disable=SC2028 1386 echo "INSERT INTO exchange.kyc_targets 1387 (h_normalized_payto, is_wallet) 1388 VALUES 1389 ('\x5555555555555555555555555555555555555555555555555555555555555555', 1390 FALSE); 1391 INSERT INTO exchange.legitimization_outcomes 1392 (h_payto, decision_time, expiration_time, to_investigate) 1393 VALUES 1394 ('\x5555555555555555555555555555555555555555555555555555555555555555', 1395 1000000000000000, 2000000000000000, FALSE);" \ 1396 | psql -Aqt "$DB" 1397 echo "DONE" 1398 1399 run_audit 1400 check_auditor_running 1401 1402 echo -n "Checking that the unjustified outcome was flagged... " 1403 check_report_any \ 1404 "row-inconsistencies" \ 1405 "diagnostic" \ 1406 "KYC rules in force without an AML decision, a legitimization process or an expired predecessor" 1407 echo -n "Checking that it was blamed on the right table... " 1408 check_report_any \ 1409 "row-inconsistencies" \ 1410 "row_table" "legitimization_outcomes" 1411 1412 # Only the injected row is bad. The outcomes the reference database 1413 # earned honestly are each accounted for by a legitimization process or 1414 # an AML decision, and reporting those too would make the check useless. 1415 echo -n "Checking that the honest outcomes were left alone... " 1416 BAD=$(jq -r '[.records[] 1417 | select(.row_table == "legitimization_outcomes")] 1418 | length' \ 1419 < "${MY_TMP_DIR}/row-inconsistencies.json") 1420 if [ "$BAD" != "1" ] 1421 then 1422 exit_fail "wanted exactly 1 bad legitimization outcome, got $BAD" 1423 fi 1424 echo "PASS" 1425 1426 full_reload 1427 cleanup 1428 } 1429 1430 1431 # Releasing the funds must retract the alert, not just stop adding to it. 1432 # The exchange announces the release by writing the `wire_out' row, which is 1433 # the only thing that distinguishes "paid" from "decided what to pay"; so 1434 # that is what this test injects. Nothing else changes: the deposits, the 1435 # aggregation_tracking rows and the open KYC measure all stay exactly as they 1436 # were, which is the point -- the hold must end because the money moved, and 1437 # for no other reason. 1438 function test_14() { 1439 1440 echo "===========14: releasing a withheld payout===========" 1441 echo -n "Modifying database: " 1442 # Take the transfer straight from the transient the exchange parked it in, 1443 # so that the test does not have to know the randomly drawn wtid. 1444 echo "INSERT INTO exchange.wire_out 1445 (execution_date, wtid_raw, wire_target_h_payto, 1446 exchange_account_section, amount) 1447 SELECT 1785869200000000 1448 ,t.wtid_raw 1449 ,t.wire_target_h_payto 1450 ,t.exchange_account_section 1451 ,t.amount 1452 FROM exchange.aggregation_transient t;" \ 1453 | psql -Aqt "$DB" 1454 echo "DONE" 1455 1456 run_audit 1457 check_auditor_running 1458 1459 echo -n "Checking that the hold was retracted... " 1460 check_no_report "aml-holds" 1461 echo -n "Checking that the AML total went back to zero... " 1462 check_balance \ 1463 "total_aml_hold" \ 1464 "TESTKUDOS:0" \ 1465 "Released funds are still counted as withheld" 1466 echo -n "Checking that they did not turn into a lag instead... " 1467 check_balance \ 1468 "total_transfer_lag" \ 1469 "TESTKUDOS:0" \ 1470 "Released funds were reclassified rather than dropped" 1471 echo -n "Checking that they were not parked as a small aggregate... " 1472 check_balance \ 1473 "total_small_aggregate" \ 1474 "TESTKUDOS:0" \ 1475 "Released funds were reclassified rather than dropped" 1476 1477 full_reload 1478 cleanup 1479 } 1480 1481 1482 # What the exchange *claims* decides which balance a hold lands in; whether the 1483 # claim holds up is a separate question, and this is the test that they are 1484 # separate. Here the measure the exchange named is carried through to a 1485 # decision while the exchange goes on withholding the money and goes on saying 1486 # it is withholding it for that measure. 1487 # 1488 # The money must therefore stay in `total_aml_hold' -- the auditor does not get 1489 # to overrule the exchange about its own reasons -- and the auditor must report 1490 # a row_inconsistency against `aggregation_deferrals', because the one part of 1491 # the claim it can check has stopped being true. Deciding what to make of that 1492 # is a job for the humans reading the finding. 1493 function test_15() { 1494 1495 echo "===========15: a hold whose KYC measure was satisfied===========" 1496 echo -n "Modifying database: " 1497 # Carry the measure that is blocking the payout through to a decision, the 1498 # way the exchange would have if the merchant had filled in the form: a 1499 # process referring to the measure, and an outcome for the account decided 1500 # no earlier than that process started. `decision_time' is rounded up to a 1501 # whole second because the exchange stores it rounded and the AML helper 1502 # reads it as a timestamp. 1503 echo "INSERT INTO exchange.legitimization_processes 1504 (h_payto, start_time, expiration_time, provider_name, 1505 legitimization_measure_serial_id, measure_index) 1506 SELECT kt.h_normalized_payto 1507 ,lm.start_time 1508 ,0 1509 ,'full_name_and_birthdate' 1510 ,lm.legitimization_measure_serial_id 1511 ,0 1512 FROM exchange.aggregation_transient t 1513 JOIN exchange.wire_targets wt 1514 ON (wt.wire_target_h_payto=t.wire_target_h_payto) 1515 JOIN exchange.kyc_targets kt 1516 ON (kt.h_normalized_payto=wt.h_normalized_payto) 1517 JOIN exchange.legitimization_measures lm 1518 USING (access_token); 1519 INSERT INTO exchange.legitimization_outcomes 1520 (h_payto, decision_time, expiration_time, to_investigate) 1521 SELECT kt.h_normalized_payto 1522 ,(lm.start_time/1000000+1)*1000000 1523 ,2000000000000000 1524 ,FALSE 1525 FROM exchange.aggregation_transient t 1526 JOIN exchange.wire_targets wt 1527 ON (wt.wire_target_h_payto=t.wire_target_h_payto) 1528 JOIN exchange.kyc_targets kt 1529 ON (kt.h_normalized_payto=wt.h_normalized_payto) 1530 JOIN exchange.legitimization_measures lm 1531 USING (access_token);" \ 1532 | psql -Aqt "$DB" 1533 echo "DONE" 1534 1535 run_audit 1536 check_auditor_running 1537 1538 echo -n "Checking that the exchange is still taken at its word... " 1539 check_balance \ 1540 "total_aml_hold" \ 1541 "$WITHHELD" \ 1542 "Auditor reclassified a hold the exchange still attributes to KYC" 1543 echo -n "Checking that it did not become an unexplained lag... " 1544 check_balance \ 1545 "total_transfer_lag" \ 1546 "TESTKUDOS:0" \ 1547 "Auditor overruled the exchange's stated reason" 1548 echo -n "Checking that the hold still names the claimed reason... " 1549 check_report "aml-holds" "deferral_reason" "KYC" 1550 echo -n "Checking that it still names the blocked account... " 1551 check_report_any \ 1552 "aml-holds" \ 1553 "account" \ 1554 "$BLOCKED_ACCOUNT" 1555 echo -n "Checking that the stale claim was reported... " 1556 check_report_any \ 1557 "row-inconsistencies" \ 1558 "row_table" "aggregation_deferrals" 1559 1560 full_reload 1561 cleanup 1562 } 1563 1564 1565 # The other thing an exchange can lawfully be sitting on: an aggregate that 1566 # would not survive its own wire fee. That is not an irregularity and must not 1567 # read as one, so it gets a balance of its own rather than being lumped in with 1568 # payouts nobody has explained. 1569 # 1570 # The reference database has no such aggregate -- every payout in it is worth 1571 # making -- so the test restates the reason on the one deferral it does have. 1572 # Reason 1 is TALER_EXCHANGEDB_DR_AMOUNT_TOO_SMALL, 2 is ..._DR_KYC. Note that 1573 # this leaves the exchange's `aggregation_transient' still naming a 1574 # legitimization requirement, so the -i cross-check must notice that the two 1575 # no longer agree. 1576 function test_16() { 1577 1578 echo "===========16: an aggregate too small to pay out===========" 1579 echo -n "Modifying database: " 1580 # Only the transfer that is still outstanding; the cleared merchant's 1581 # deferral is on file too, but it was overtaken by its wire_out row. 1582 echo "UPDATE exchange.aggregation_deferrals d 1583 SET deferral_reason=1 1584 ,legitimization_requirement_serial_id=0 1585 WHERE NOT EXISTS ( 1586 SELECT 1 FROM exchange.wire_out w 1587 WHERE w.wtid_raw=d.wtid_raw);" \ 1588 | psql -Aqt "$DB" 1589 echo "DONE" 1590 1591 run_audit 1592 check_auditor_running 1593 1594 echo -n "Checking that the funds are booked as a small aggregate... " 1595 check_balance \ 1596 "total_small_aggregate" \ 1597 "$WITHHELD" \ 1598 "Aggregate below the wire fee was not booked as such" 1599 echo -n "Checking that they are no longer an AML hold... " 1600 check_balance \ 1601 "total_aml_hold" \ 1602 "TESTKUDOS:0" \ 1603 "Aggregate below the wire fee still counted as withheld for KYC" 1604 echo -n "Checking that they are not an unexplained lag either... " 1605 check_balance \ 1606 "total_transfer_lag" \ 1607 "TESTKUDOS:0" \ 1608 "Aggregate below the wire fee read as an unexplained delay" 1609 echo -n "Checking that the hold names the new reason... " 1610 check_report "aml-holds" "deferral_reason" "AMOUNT_TOO_SMALL" 1611 echo -n "Checking that the contradiction with the transient was reported... " 1612 check_report_any \ 1613 "row-inconsistencies" \ 1614 "row_table" "aggregation_transient" 1615 1616 full_reload 1617 cleanup 1618 } 1619 1620 1621 # *************** Main test loop starts here ************** 1622 1623 1624 # Run all the tests against the database given in $1. 1625 # Sets $fail to 0 on success, non-zero on failure. 1626 function check_with_database() 1627 { 1628 BASEDB="$1" 1629 CONF="$1.conf" 1630 export CONF 1631 echo "Running test suite with database $BASEDB using configuration $CONF" 1632 MASTER_PRIV_FILE="${BASEDB}.mpriv" 1633 taler-exchange-config \ 1634 -f \ 1635 -c "${CONF}" \ 1636 -s exchange-offline \ 1637 -o MASTER_PRIV_FILE \ 1638 -V "${MASTER_PRIV_FILE}" 1639 1640 # Load database 1641 full_reload 1642 1643 # Run test suite 1644 fail=0 1645 for i in $TESTS 1646 do 1647 "test_$i" 1648 if test 0 != $fail 1649 then 1650 break 1651 fi 1652 done 1653 echo "Cleanup (disabled, leaving database $DB behind)" 1654 # dropdb $DB 1655 } 1656 1657 # When the script is not run as root, setup a temporary directory for the 1658 # postgres database. 1659 # Sets PGHOST accordingly to the freshly created socket. 1660 function perform_initdb() { 1661 # Available directly in path? 1662 INITDB_BIN=$(command -v initdb) || true 1663 if [[ -n "$INITDB_BIN" ]]; then 1664 echo " FOUND (in path) at $INITDB_BIN" 1665 else 1666 HAVE_INITDB=$(find /usr -name "initdb" 2> /dev/null \ 1667 | head -1 2> /dev/null \ 1668 | grep postgres) \ 1669 || exit_skip " MISSING" 1670 echo " FOUND at $(dirname "$HAVE_INITDB")" 1671 INITDB_BIN=$(echo "$HAVE_INITDB" | grep bin/initdb | grep postgres | sort -n | tail -n1) 1672 fi 1673 POSTGRES_PATH=$(dirname "$INITDB_BIN") 1674 1675 TMPDIR="$MY_TMP_DIR/postgres" 1676 mkdir -p "$TMPDIR" 1677 echo -n "Setting up Postgres DB at $TMPDIR ..." 1678 $INITDB_BIN \ 1679 --no-sync \ 1680 --auth=trust \ 1681 -D "${TMPDIR}" \ 1682 > "${MY_TMP_DIR}/postgres-dbinit.log" \ 1683 2> "${MY_TMP_DIR}/postgres-dbinit.err" \ 1684 || { 1685 echo "FAILED!" 1686 echo "Last entries in ${MY_TMP_DIR}/postgres-dbinit.err:" 1687 tail "${MY_TMP_DIR}/postgres-dbinit.err" 1688 exit 1 1689 } 1690 echo "DONE" 1691 1692 # Once we move to PG16, we can use: 1693 # --set listen_addresses='' \ 1694 # --set fsync=off \ 1695 # --set max_wal_senders=0 \ 1696 # --set synchronous_commit=off \ 1697 # --set wal_level=minimal \ 1698 # --set unix_socket_directories="${TMPDIR}/sockets" \ 1699 1700 1701 SOCKETDIR="${TMPDIR}/sockets" 1702 mkdir "${SOCKETDIR}" 1703 1704 echo -n "Launching Postgres service" 1705 1706 cat - >> "$TMPDIR/postgresql.conf" <<EOF 1707 unix_socket_directories='${TMPDIR}/sockets' 1708 fsync=off 1709 max_wal_senders=0 1710 synchronous_commit=off 1711 wal_level=minimal 1712 listen_addresses='' 1713 EOF 1714 1715 grep -v host \ 1716 < "$TMPDIR/pg_hba.conf" \ 1717 > "$TMPDIR/pg_hba.conf.new" 1718 mv "$TMPDIR/pg_hba.conf.new" "$TMPDIR/pg_hba.conf" 1719 "${POSTGRES_PATH}/pg_ctl" \ 1720 -D "$TMPDIR" \ 1721 -l "${MY_TMP_DIR}/postgres.log" \ 1722 start \ 1723 > "${MY_TMP_DIR}/postgres-start.log" \ 1724 2> "${MY_TMP_DIR}/postgres-start.err" 1725 echo " DONE" 1726 PGHOST="$TMPDIR/sockets" 1727 export PGHOST 1728 } 1729 1730 1731 # *************** Main logic starts here ************** 1732 1733 # ####### Setup globals ###### 1734 # Postgres database to use (must match configuration file) 1735 export DB="auditor-basedb" 1736 1737 # test required commands exist 1738 echo "Testing for jq" 1739 jq -h > /dev/null || exit_skip "jq required" 1740 echo "Testing for taler-merchant-config" 1741 taler-merchant-config -h > /dev/null || exit_skip "taler-merchant-config required" 1742 echo "Testing for taler-merchant-httpd" 1743 taler-merchant-httpd -h > /dev/null || exit_skip "taler-merchant-httpd required" 1744 echo "Testing for faketime" 1745 faketime -h > /dev/null || exit_skip "faketime required" 1746 # NOTE: really check for all three libeufin commands? 1747 echo "Testing for libeufin" 1748 libeufin-bank --help >/dev/null 2> /dev/null </dev/null || exit_skip "libeufin required" 1749 echo "Testing for taler-wallet-cli" 1750 taler-wallet-cli -h >/dev/null </dev/null 2>/dev/null || exit_skip "taler-wallet-cli required" 1751 1752 1753 echo -n "Testing for Postgres" 1754 1755 MY_TMP_DIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX) 1756 echo "Using $MY_TMP_DIR for logging and temporary data" 1757 1758 # If run as root, simply use the running postgres instance. 1759 # Otherwise create a temporary storage space for postgres. 1760 [ $(id -u) == 0 ] || perform_initdb 1761 1762 MYDIR="${MY_TMP_DIR}/basedb" 1763 mkdir -p "${MYDIR}" 1764 1765 if [ -z ${REUSE_BASEDB_DIR+x} ] 1766 then 1767 echo "Generating fresh database at $MYDIR" 1768 1769 if faketime -f '-1 d' ./generate-kyc-basedb.sh -d "$MYDIR/$DB" 1770 then 1771 echo -n "Reset 'auditor-basedb' database at ${PGHOST:-} ..." 1772 dropdb --if-exists "auditor-basedb" > /dev/null 2> /dev/null || true 1773 createdb "auditor-basedb" || exit_skip "Could not create database '$BASEDB' at ${PGHOST:-}" 1774 echo " DONE" 1775 else 1776 echo "Generation failed" 1777 exit 1 1778 fi 1779 echo "To reuse this database in the future, use:" 1780 echo "export REUSE_BASEDB_DIR=$MY_TMP_DIR" 1781 else 1782 echo "Reusing existing database from ${REUSE_BASEDB_DIR}" 1783 cp -r "${REUSE_BASEDB_DIR}/basedb"/* "${MYDIR}/" 1784 fi 1785 1786 check_with_database "$MYDIR/$DB" 1787 if [ "$fail" != "0" ] 1788 then 1789 exit "$fail" 1790 fi 1791 1792 if [ -z "${REUSE_BASEDB_DIR+x}" ] 1793 then 1794 echo "Run 'export REUSE_BASEDB_DIR=${MY_TMP_DIR}' to re-run tests against the same database" 1795 fi 1796 1797 exit 0