generate-kyc-basedb.sh (16919B)
1 #!/bin/bash 2 # This file is in the public domain. 3 # 4 # Script to generate the reference database for the KYC/AML auditor tests 5 # from a 'correct' interaction between exchange, wallet, merchant and bank 6 # with KYC enabled. 7 # 8 # Creates "$1.sql", "$1.conf" and "$1.mpriv". 9 # 10 # Unlike generate-auditor-basedb.sh the resulting database deliberately 11 # contains *both* KYC outcomes the auditor can observe. Two merchant 12 # instances are paid the same amount, and both prove that they own their 13 # bank account with a KYCAUTH wire transfer, but only one of them then 14 # passes the KYC form: 15 # 16 # - instance 'default' (account DE474361) passes, so the funds it is 17 # owed are aggregated and wired out normally; 18 # - instance 'blocked' (account DE61109451) does not, so the exchange 19 # aggregates its coins but parks the payout in aggregation_transient 20 # and never wires it anywhere. 21 # 22 # The second case is what makes this database worth auditing: the exchange 23 # is holding money it has not paid out, and test-kyc.sh pins down what the 24 # auditor does and does not see of that (see issues.txt, issue 9). 25 # 26 # Requires the wallet CLI to be installed and in the path. Furthermore, the 27 # user running this script must be Postgres superuser and be allowed to 28 # create/drop databases. 29 # 30 set -eu 31 32 # The auditor tests inject faults into specific rows of the generated 33 # database, so the wallet must pick the same coins and make the same 34 # transactions on every run. The coin selection introduced after 2024 is 35 # not deterministic in that way; 'legacy-2024' restores the old behaviour 36 # (see https://bugs.gnunet.org/view.php?id=11272). 37 export TALER_WALLET_COINSEL="legacy-2024" 38 39 . setup.sh 40 41 CONF="generate-kyc-basedb.conf" 42 # Parse command-line options 43 while getopts ':c:d:h' OPTION; do 44 case "$OPTION" in 45 c) 46 CONF="$OPTARG" 47 ;; 48 d) 49 BASEDB="$OPTARG" 50 ;; 51 h) 52 echo 'Supported options:' 53 # shellcheck disable=SC2016 54 echo ' -c $CONF -- set configuration' 55 # shellcheck disable=SC2016 56 echo ' -d $DB -- set database name' 57 ;; 58 ?) 59 exit_fail "Unrecognized command line option" 60 ;; 61 esac 62 done 63 64 # Where do we write the result? 65 if [ ! -v BASEDB ] 66 then 67 exit_fail "-d option required" 68 fi 69 70 echo -n "Testing for curl ..." 71 curl --help >/dev/null </dev/null || exit_skip " MISSING" 72 echo " FOUND" 73 echo -n "Testing for jq ..." 74 jq -h >/dev/null </dev/null || exit_skip " MISSING" 75 echo " FOUND" 76 77 # The bank account of the merchant instance that passes KYC, and the one 78 # of the instance that stays blocked. 79 CLEARED_PAYTO="payto://iban/DE474361?receiver-name=Merchant43" 80 BLOCKED_PAYTO="payto://iban/DE61109451?receiver-name=Blocked44" 81 82 # reset database 83 echo -n "Reset 'auditor-basedb' database at ${PGHOST:-} ..." 84 dropdb --if-exists "auditor-basedb" > /dev/null 2> /dev/null || true 85 createdb "auditor-basedb" || exit_skip "Could not create database '$BASEDB' at ${PGHOST:-}" 86 echo " DONE" 87 88 # Launch exchange, merchant and bank. 89 setup -c "$CONF" \ 90 -abemw \ 91 -d "iban" 92 93 # obtain key configuration data 94 EXCHANGE_URL=$(taler-exchange-config -c "$CONF" -s EXCHANGE -o BASE_URL) 95 MERCHANT_PORT=$(taler-merchant-config -c "$CONF" -s MERCHANT -o PORT) 96 MERCHANT_URL="http://localhost:${MERCHANT_PORT}/" 97 BANK_PORT=$(taler-exchange-config -c "$CONF" -s BANK -o HTTP_PORT) 98 BANK_URL="http://localhost:${BANK_PORT}/" 99 WIRE_GATEWAY_URL=$(taler-exchange-config -c "${CONF}.edited" \ 100 -s exchange-accountcredentials-1 \ 101 -o WIRE_GATEWAY_URL) 102 103 echo -n "Checking setup worked ..." 104 wget \ 105 --tries=1 \ 106 --timeout=1 \ 107 "${EXCHANGE_URL}config" \ 108 -o /dev/null \ 109 -O /dev/null >/dev/null 110 echo "DONE" 111 112 export MERCHANT_URL 113 114 # Create merchant instance $1 with bank account $2, and a matching bank 115 # account at libeufin named $3. 116 function setup_instance() { 117 local id="$1" 118 local payto="$2" 119 local user="$3" 120 local base 121 122 if [ "$id" = "admin" ] 123 then 124 base="${MERCHANT_URL}" 125 else 126 base="${MERCHANT_URL}instances/${id}/" 127 fi 128 129 echo -n "Setting up merchant instance ${id} ..." 130 curl -s -o /dev/null \ 131 -H "Content-Type: application/json" \ 132 -X POST \ 133 -d "$(jq -n --arg id "$id" \ 134 '{auth: {method: "external"}, 135 id: $id, 136 name: $id, 137 address: {}, 138 jurisdiction: {}, 139 default_max_wire_fee: "TESTKUDOS:1", 140 default_max_deposit_fee: "TESTKUDOS:1", 141 default_wire_fee_amortization: 1, 142 default_wire_transfer_delay: {d_us: 0}, 143 default_pay_delay: {d_us: 3600000000}, 144 use_stefan: false}')" \ 145 "${MERCHANT_URL}management/instances" 146 echo " DONE" 147 148 echo -n "Setting up merchant account for ${id} ..." 149 STATUS=$(curl -H "Content-Type: application/json" -X POST \ 150 "${base}private/accounts" \ 151 -d "$(jq -n --arg p "$payto" '{payto_uri: $p}')" \ 152 -w "%{http_code}" -s -o /dev/null) 153 if [ "$STATUS" != "200" ] 154 then 155 exit_fail "Expected 200 OK from ${base}private/accounts. Got: $STATUS" 156 fi 157 echo " DONE" 158 159 echo -n "Setting up libeufin account ${user} ..." 160 libeufin-bank create-account \ 161 --config="${CONF}" \ 162 --name="${user}" \ 163 --username="${user}" \ 164 --password="password" \ 165 --payto_uri="${payto}" \ 166 > /dev/null 167 echo " DONE" 168 } 169 170 # ---------------------------------------------------------------------- 171 # With ENABLE_KYC the exchange refuses to say anything about an account it 172 # has never seen, and the merchant in turn refuses to create orders (451). 173 # A merchant proves that it owns its bank account by having a small KYCAUTH 174 # wire transfer made from it, quoting its own instance public key. Those 175 # transfers are also the only KYC-specific wire credits the auditor knows 176 # about, so the reference database needs them anyway. 177 # 178 # Sets ACCESS_TOKEN to the KYC access token the exchange handed out for the 179 # account of instance $1. 180 # ---------------------------------------------------------------------- 181 182 function kycauth_instance() { 183 local id="$1" 184 local base 185 local filter 186 local pub 187 local payto 188 local status 189 local n 190 191 if [ "$id" = "admin" ] 192 then 193 base="${MERCHANT_URL}" 194 else 195 base="${MERCHANT_URL}instances/${id}/" 196 fi 197 # An instance also carries the default accounts of the public demo 198 # exchanges, so pick out the entry that describes *our* exchange. 199 filter=".kyc_data[] | select(.exchange_url == \"${EXCHANGE_URL}\")" 200 201 echo -n "Reading public key of instance ${id} ..." 202 pub=$(curl -s "${base}private" | jq -r '.merchant_pub') 203 if [ -z "$pub" ] || [ "$pub" = "null" ] 204 then 205 exit_fail "Could not determine public key of instance ${id}" 206 fi 207 echo " ${pub}" 208 209 echo -n "Waiting for instance ${id} to talk to the exchange ..." 210 payto="" 211 for n in $(seq 1 100) 212 do 213 echo -n "." 214 sleep 0.5 215 KYC_JSON=$(curl -s "${base}private/kyc") || continue 216 status=$(echo "$KYC_JSON" | jq -r "${filter} | .exchange_http_status // empty") 217 # 0 means "exchange not reachable yet" 218 if [ -z "$status" ] || [ "$status" = "0" ] 219 then 220 continue 221 fi 222 payto=$(echo "$KYC_JSON" | jq -r "${filter} | .payto_uri") 223 break 224 done 225 if [ -z "$payto" ] || [ "$payto" = "null" ] 226 then 227 echo "$KYC_JSON" 228 exit_fail "Instance ${id} never obtained a KYC status from the exchange" 229 fi 230 echo " DONE (status $status, account $payto)" 231 232 echo -n "Making the KYCAUTH wire transfer for ${id} ..." 233 status=$(curl -s -o "kycauth-${id}.json" -w "%{http_code}" \ 234 -u "admin:secret-password" \ 235 -H "Content-Type: application/json" \ 236 -X POST \ 237 -d "$(jq -n \ 238 --arg pub "$pub" \ 239 --arg debit "$payto" \ 240 '{account_pub: $pub, 241 amount: "TESTKUDOS:0.1", 242 debit_account: $debit}')" \ 243 "${WIRE_GATEWAY_URL}admin/add-kycauth") 244 if [ "$status" != "200" ] 245 then 246 cat "kycauth-${id}.json" 247 exit_fail "Expected 200 OK from admin/add-kycauth. Got: $status" 248 fi 249 echo " DONE" 250 251 echo -n "Waiting for the exchange to accept the KYCAUTH of ${id} ..." 252 for n in $(seq 1 100) 253 do 254 echo -n "." 255 sleep 0.5 256 KYC_JSON=$(curl -s "${base}private/kyc") || continue 257 status=$(echo "$KYC_JSON" | jq -r "${filter} | .exchange_http_status // empty") 258 if [ "$status" = "200" ] 259 then 260 break 261 fi 262 done 263 if [ "$status" != "200" ] 264 then 265 echo "$KYC_JSON" 266 exit_fail "Exchange never accepted the KYCAUTH transfer of ${id}" 267 fi 268 ACCESS_TOKEN=$(echo "$KYC_JSON" | jq -r "${filter} | .access_token // empty") 269 if [ -z "$ACCESS_TOKEN" ] 270 then 271 echo "$KYC_JSON" | jq "${filter}" 272 exit_fail "Instance ${id} did not receive a KYC access token" 273 fi 274 echo " DONE" 275 } 276 277 # Satisfy KYC measure M1 for the account behind access token $1 by 278 # submitting the form that KYC-CHECK-C1 asks for. 279 # taler-exchange-helper-measure-test-form then raises every limit to 280 # TESTKUDOS:1000, which un-blocks the account. 281 function submit_kyc_form() { 282 local token="$1" 283 local id 284 local n 285 286 # The measure is only instantiated when someone asks for it, so poll 287 # until the exchange offers something to do. 288 echo -n "Fetching KYC requirements ..." 289 for n in $(seq 1 100) 290 do 291 echo -n "." 292 curl -s -o kyc-info.json "${EXCHANGE_URL}kyc-info/${token}" 293 id=$(jq -r '.requirements[]? | select(.id != null) | .id' < kyc-info.json \ 294 | head -n1) 295 if [ -n "$id" ] 296 then 297 break 298 fi 299 sleep 0.5 300 done 301 if [ -z "$id" ] 302 then 303 cat kyc-info.json 304 exit_fail "Exchange did not offer a KYC requirement to fulfill" 305 fi 306 echo " DONE (requirement $id)" 307 308 echo -n "Submitting the KYC form ..." 309 STATUS=$(curl -s -o kyc-upload.json -w "%{http_code}" \ 310 -H "Content-Type: application/json" \ 311 -X POST \ 312 -d '{"FULL_NAME":"Merchant Fourtythree", 313 "DATE_OF_BIRTH":"2000-01-01", 314 "FORM_ID":"full_name_and_birthdate"}' \ 315 "${EXCHANGE_URL}kyc-upload/${id}") 316 if [ "$STATUS" != "204" ] 317 then 318 cat kyc-upload.json 319 exit_fail "Expected 204 No Content from kyc-upload. Got: $STATUS" 320 fi 321 echo " DONE" 322 } 323 324 # Run the aggregator once, logging to aggregator-$1.log. 325 # 326 # Note the deliberate absence of '-y' (--kyc-off): that option makes the 327 # aggregator wire funds out without consulting KYC at all, which is exactly 328 # what this reference database must not do. 329 # 330 # The '-T' travels two hours forward so that the deposits' wire deadlines 331 # have passed -- the wallet's integration-test order sets them a quarter of 332 # an hour out, and simply moving them into the past instead makes the 333 # aggregator spin forever on the KYC-blocked transient aggregation. The 334 # price is that the wire_out rows come out stamped two hours ahead of when 335 # the bank actually moved the money, which the auditor would rightly report 336 # as a minor row inconsistency; the caller fixes the timestamps up again 337 # once the transfer has been made. 338 function run_aggregator() { 339 echo -n "Running exchange aggregator ($1) ..." 340 taler-exchange-aggregator \ 341 -T 7200000000 \ 342 -L INFO \ 343 -t \ 344 -c "${CONF}.edited" \ 345 2> "aggregator-$1.log" \ 346 || exit_fail "aggregator failed" 347 echo " DONE" 348 } 349 350 # Pay TESTKUDOS:4 to the merchant instance $1, withdrawing TESTKUDOS:10 351 # first. Uses its own wallet database so that coin selection does not 352 # depend on what an earlier instance already spent. 353 function pay_instance() { 354 local id="$1" 355 local base 356 local db="wallet-${id}.wdb" 357 local wlog="taler-wallet-cli-${id}.log" 358 359 if [ "$id" = "admin" ] 360 then 361 base="${MERCHANT_URL}" 362 else 363 base="${MERCHANT_URL}instances/${id}/" 364 fi 365 366 rm -f "$db" 367 echo -n "Running wallet against instance ${id} ..." 368 taler-wallet-cli \ 369 --no-throttle \ 370 --wallet-db="$db" \ 371 api \ 372 --expect-success \ 373 'runIntegrationTest' \ 374 "$(jq -n ' 375 { 376 amountToSpend: "TESTKUDOS:4", 377 amountToWithdraw: "TESTKUDOS:10", 378 corebankApiBaseUrl: $BANK_URL, 379 exchangeBaseUrl: $EXCHANGE_URL, 380 merchantBaseUrl: $MERCHANT_URL, 381 }' \ 382 --arg MERCHANT_URL "$base" \ 383 --arg EXCHANGE_URL "$EXCHANGE_URL" \ 384 --arg BANK_URL "$BANK_URL" 385 )" &> "$wlog" || { 386 echo " FAILED! Last lines from $wlog:" 387 tail "$wlog" 388 exit 2 389 } 390 echo " DONE" 391 taler-wallet-cli --wallet-db="$db" run-until-done &>> "$wlog" 392 } 393 394 # ---------------------------------------------------------------------- 395 # Both instances prove that they own their bank account -- without that 396 # the exchange refuses their deposits outright -- and are then paid the 397 # same TESTKUDOS:4. Both amounts exceed the AGGREGATE threshold, so the 398 # exchange accepts the coins but may not wire either of them out yet. 399 # ---------------------------------------------------------------------- 400 401 setup_instance "admin" "$CLEARED_PAYTO" "Merchant43" 402 kycauth_instance "admin" 403 CLEARED_ACCESS_TOKEN="$ACCESS_TOKEN" 404 405 setup_instance "blocked" "$BLOCKED_PAYTO" "Blocked44" 406 kycauth_instance "blocked" 407 BLOCKED_ACCESS_TOKEN="$ACCESS_TOKEN" 408 409 pay_instance "admin" 410 pay_instance "blocked" 411 412 # The KYC rule only fires once the exchange actually tries to aggregate, 413 # so run the aggregator before there is a measure to satisfy. 414 run_aggregator "first" 415 416 echo -n "Checking that no wire transfer happened yet ..." 417 WIRE_OUT=$(psql -Aqt "auditor-basedb" \ 418 -c "SELECT COUNT(*) FROM exchange.wire_out;") 419 if [ "$WIRE_OUT" != "0" ] 420 then 421 exit_fail "KYC should have blocked every aggregation, but got ${WIRE_OUT} wire_out rows" 422 fi 423 echo " DONE" 424 425 # ---------------------------------------------------------------------- 426 # Only instance 'admin' passes the form. After that its funds may be 427 # wired out; instance 'blocked' stays blocked, and the payout it is owed 428 # stays parked in aggregation_transient for good. 429 # ---------------------------------------------------------------------- 430 431 submit_kyc_form "$CLEARED_ACCESS_TOKEN" 432 433 run_aggregator "second" 434 435 echo -n "Running exchange transfer ..." 436 taler-exchange-transfer \ 437 -L INFO \ 438 -t \ 439 -c "${CONF}.edited" \ 440 2> transfer.log \ 441 || exit_fail "transfer failed" 442 echo " DONE" 443 444 445 # Undo the side effect of the aggregator's time travel: the exchange 446 # recorded the wire transfer as having been executed two hours from now, 447 # while the bank stamped it with the real time. The auditor compares the 448 # two and reports any disagreement of more than 15 minutes, so bring the 449 # exchange's timestamp back to reality. wire_out.execution_date carries no 450 # signature; test-auditor.sh's test 17 edits the very same field. Round to 451 # whole seconds: a GNUNET_TIME_Timestamp may not carry a sub-second part, 452 # and the helpers assert on that when they read the row back. 453 echo -n "Correcting the wire_out execution dates ..." 454 psql -Aqt "auditor-basedb" \ 455 -c "UPDATE exchange.wire_out SET execution_date=1000000*((SELECT transaction_date FROM libeufin_bank.bank_account_transactions WHERE direction='debit' ORDER BY bank_transaction_id DESC LIMIT 1)/1000000);" \ 456 > /dev/null 457 echo " DONE" 458 459 echo -n "Checking that exactly the cleared account was paid ..." 460 WIRE_OUT=$(psql -Aqt "auditor-basedb" \ 461 -c "SELECT COUNT(*) FROM exchange.wire_out;") 462 if [ "$WIRE_OUT" != "1" ] 463 then 464 exit_fail "Expected exactly one wire transfer, got ${WIRE_OUT}" 465 fi 466 echo " DONE" 467 468 echo "KYC access token of the blocked account: ${BLOCKED_ACCESS_TOKEN}" 469 470 # Dump database 471 mkdir -p "$(dirname "$BASEDB")" 472 473 echo "Dumping database ${BASEDB}.sql" 474 pg_dump -O "auditor-basedb" | sed -e '/AS integer/d' > "${BASEDB}.sql" 475 cp "${CONF}.edited" "${BASEDB}.conf" 476 cp "$(taler-exchange-config -c "${CONF}.edited" -s exchange-offline -o MASTER_PRIV_FILE -f)" "${BASEDB}.mpriv" 477 478 # clean up 479 echo -n "Final clean up ..." 480 kill -TERM "$SETUP_PID" 481 wait 482 unset SETUP_PID 483 dropdb "auditor-basedb" 484 echo " DONE" 485 486 echo "=====================================" 487 echo "Finished generation of ${BASEDB}.sql" 488 echo "=====================================" 489 490 exit 0