commit 98b9c113c1a447054767dca64371911ca4d070cb
parent 964723bcd660682afee95b53247debd6e0e9f495
Author: Christian Grothoff <christian@grothoff.org>
Date: Sat, 1 Aug 2026 00:49:33 +0200
add missing kyc generation file
Diffstat:
1 file changed, 490 insertions(+), 0 deletions(-)
diff --git a/src/auditor/generate-kyc-basedb.sh b/src/auditor/generate-kyc-basedb.sh
@@ -0,0 +1,490 @@
+#!/bin/bash
+# This file is in the public domain.
+#
+# Script to generate the reference database for the KYC/AML auditor tests
+# from a 'correct' interaction between exchange, wallet, merchant and bank
+# with KYC enabled.
+#
+# Creates "$1.sql", "$1.conf" and "$1.mpriv".
+#
+# Unlike generate-auditor-basedb.sh the resulting database deliberately
+# contains *both* KYC outcomes the auditor can observe. Two merchant
+# instances are paid the same amount, and both prove that they own their
+# bank account with a KYCAUTH wire transfer, but only one of them then
+# passes the KYC form:
+#
+# - instance 'default' (account DE474361) passes, so the funds it is
+# owed are aggregated and wired out normally;
+# - instance 'blocked' (account DE61109451) does not, so the exchange
+# aggregates its coins but parks the payout in aggregation_transient
+# and never wires it anywhere.
+#
+# The second case is what makes this database worth auditing: the exchange
+# is holding money it has not paid out, and test-kyc.sh pins down what the
+# auditor does and does not see of that (see issues.txt, issue 9).
+#
+# Requires the wallet CLI to be installed and in the path. Furthermore, the
+# user running this script must be Postgres superuser and be allowed to
+# create/drop databases.
+#
+set -eu
+
+# The auditor tests inject faults into specific rows of the generated
+# database, so the wallet must pick the same coins and make the same
+# transactions on every run. The coin selection introduced after 2024 is
+# not deterministic in that way; 'legacy-2024' restores the old behaviour
+# (see https://bugs.gnunet.org/view.php?id=11272).
+export TALER_WALLET_COINSEL="legacy-2024"
+
+. setup.sh
+
+CONF="generate-kyc-basedb.conf"
+# Parse command-line options
+while getopts ':c:d:h' OPTION; do
+ case "$OPTION" in
+ c)
+ CONF="$OPTARG"
+ ;;
+ d)
+ BASEDB="$OPTARG"
+ ;;
+ h)
+ echo 'Supported options:'
+# shellcheck disable=SC2016
+ echo ' -c $CONF -- set configuration'
+# shellcheck disable=SC2016
+ echo ' -d $DB -- set database name'
+ ;;
+ ?)
+ exit_fail "Unrecognized command line option"
+ ;;
+ esac
+done
+
+# Where do we write the result?
+if [ ! -v BASEDB ]
+then
+ exit_fail "-d option required"
+fi
+
+echo -n "Testing for curl ..."
+curl --help >/dev/null </dev/null || exit_skip " MISSING"
+echo " FOUND"
+echo -n "Testing for jq ..."
+jq -h >/dev/null </dev/null || exit_skip " MISSING"
+echo " FOUND"
+
+# The bank account of the merchant instance that passes KYC, and the one
+# of the instance that stays blocked.
+CLEARED_PAYTO="payto://iban/DE474361?receiver-name=Merchant43"
+BLOCKED_PAYTO="payto://iban/DE61109451?receiver-name=Blocked44"
+
+# reset database
+echo -n "Reset 'auditor-basedb' database at ${PGHOST:-} ..."
+dropdb --if-exists "auditor-basedb" > /dev/null 2> /dev/null || true
+createdb "auditor-basedb" || exit_skip "Could not create database '$BASEDB' at ${PGHOST:-}"
+echo " DONE"
+
+# Launch exchange, merchant and bank.
+setup -c "$CONF" \
+ -abemw \
+ -d "iban"
+
+# obtain key configuration data
+EXCHANGE_URL=$(taler-exchange-config -c "$CONF" -s EXCHANGE -o BASE_URL)
+MERCHANT_PORT=$(taler-merchant-config -c "$CONF" -s MERCHANT -o PORT)
+MERCHANT_URL="http://localhost:${MERCHANT_PORT}/"
+BANK_PORT=$(taler-exchange-config -c "$CONF" -s BANK -o HTTP_PORT)
+BANK_URL="http://localhost:${BANK_PORT}/"
+WIRE_GATEWAY_URL=$(taler-exchange-config -c "${CONF}.edited" \
+ -s exchange-accountcredentials-1 \
+ -o WIRE_GATEWAY_URL)
+
+echo -n "Checking setup worked ..."
+wget \
+ --tries=1 \
+ --timeout=1 \
+ "${EXCHANGE_URL}config" \
+ -o /dev/null \
+ -O /dev/null >/dev/null
+echo "DONE"
+
+export MERCHANT_URL
+
+# Create merchant instance $1 with bank account $2, and a matching bank
+# account at libeufin named $3.
+function setup_instance() {
+ local id="$1"
+ local payto="$2"
+ local user="$3"
+ local base
+
+ if [ "$id" = "admin" ]
+ then
+ base="${MERCHANT_URL}"
+ else
+ base="${MERCHANT_URL}instances/${id}/"
+ fi
+
+ echo -n "Setting up merchant instance ${id} ..."
+ curl -s -o /dev/null \
+ -H "Content-Type: application/json" \
+ -X POST \
+ -d "$(jq -n --arg id "$id" \
+ '{auth: {method: "external"},
+ id: $id,
+ name: $id,
+ address: {},
+ jurisdiction: {},
+ default_max_wire_fee: "TESTKUDOS:1",
+ default_max_deposit_fee: "TESTKUDOS:1",
+ default_wire_fee_amortization: 1,
+ default_wire_transfer_delay: {d_us: 0},
+ default_pay_delay: {d_us: 3600000000},
+ use_stefan: false}')" \
+ "${MERCHANT_URL}management/instances"
+ echo " DONE"
+
+ echo -n "Setting up merchant account for ${id} ..."
+ STATUS=$(curl -H "Content-Type: application/json" -X POST \
+ "${base}private/accounts" \
+ -d "$(jq -n --arg p "$payto" '{payto_uri: $p}')" \
+ -w "%{http_code}" -s -o /dev/null)
+ if [ "$STATUS" != "200" ]
+ then
+ exit_fail "Expected 200 OK from ${base}private/accounts. Got: $STATUS"
+ fi
+ echo " DONE"
+
+ echo -n "Setting up libeufin account ${user} ..."
+ libeufin-bank create-account \
+ --config="${CONF}" \
+ --name="${user}" \
+ --username="${user}" \
+ --password="password" \
+ --payto_uri="${payto}" \
+ > /dev/null
+ echo " DONE"
+}
+
+# ----------------------------------------------------------------------
+# With ENABLE_KYC the exchange refuses to say anything about an account it
+# has never seen, and the merchant in turn refuses to create orders (451).
+# A merchant proves that it owns its bank account by having a small KYCAUTH
+# wire transfer made from it, quoting its own instance public key. Those
+# transfers are also the only KYC-specific wire credits the auditor knows
+# about, so the reference database needs them anyway.
+#
+# Sets ACCESS_TOKEN to the KYC access token the exchange handed out for the
+# account of instance $1.
+# ----------------------------------------------------------------------
+
+function kycauth_instance() {
+ local id="$1"
+ local base
+ local filter
+ local pub
+ local payto
+ local status
+ local n
+
+ if [ "$id" = "admin" ]
+ then
+ base="${MERCHANT_URL}"
+ else
+ base="${MERCHANT_URL}instances/${id}/"
+ fi
+ # An instance also carries the default accounts of the public demo
+ # exchanges, so pick out the entry that describes *our* exchange.
+ filter=".kyc_data[] | select(.exchange_url == \"${EXCHANGE_URL}\")"
+
+ echo -n "Reading public key of instance ${id} ..."
+ pub=$(curl -s "${base}private" | jq -r '.merchant_pub')
+ if [ -z "$pub" ] || [ "$pub" = "null" ]
+ then
+ exit_fail "Could not determine public key of instance ${id}"
+ fi
+ echo " ${pub}"
+
+ echo -n "Waiting for instance ${id} to talk to the exchange ..."
+ payto=""
+ for n in $(seq 1 100)
+ do
+ echo -n "."
+ sleep 0.5
+ KYC_JSON=$(curl -s "${base}private/kyc") || continue
+ status=$(echo "$KYC_JSON" | jq -r "${filter} | .exchange_http_status // empty")
+ # 0 means "exchange not reachable yet"
+ if [ -z "$status" ] || [ "$status" = "0" ]
+ then
+ continue
+ fi
+ payto=$(echo "$KYC_JSON" | jq -r "${filter} | .payto_uri")
+ break
+ done
+ if [ -z "$payto" ] || [ "$payto" = "null" ]
+ then
+ echo "$KYC_JSON"
+ exit_fail "Instance ${id} never obtained a KYC status from the exchange"
+ fi
+ echo " DONE (status $status, account $payto)"
+
+ echo -n "Making the KYCAUTH wire transfer for ${id} ..."
+ status=$(curl -s -o "kycauth-${id}.json" -w "%{http_code}" \
+ -u "admin:secret-password" \
+ -H "Content-Type: application/json" \
+ -X POST \
+ -d "$(jq -n \
+ --arg pub "$pub" \
+ --arg debit "$payto" \
+ '{account_pub: $pub,
+ amount: "TESTKUDOS:0.1",
+ debit_account: $debit}')" \
+ "${WIRE_GATEWAY_URL}admin/add-kycauth")
+ if [ "$status" != "200" ]
+ then
+ cat "kycauth-${id}.json"
+ exit_fail "Expected 200 OK from admin/add-kycauth. Got: $status"
+ fi
+ echo " DONE"
+
+ echo -n "Waiting for the exchange to accept the KYCAUTH of ${id} ..."
+ for n in $(seq 1 100)
+ do
+ echo -n "."
+ sleep 0.5
+ KYC_JSON=$(curl -s "${base}private/kyc") || continue
+ status=$(echo "$KYC_JSON" | jq -r "${filter} | .exchange_http_status // empty")
+ if [ "$status" = "200" ]
+ then
+ break
+ fi
+ done
+ if [ "$status" != "200" ]
+ then
+ echo "$KYC_JSON"
+ exit_fail "Exchange never accepted the KYCAUTH transfer of ${id}"
+ fi
+ ACCESS_TOKEN=$(echo "$KYC_JSON" | jq -r "${filter} | .access_token // empty")
+ if [ -z "$ACCESS_TOKEN" ]
+ then
+ echo "$KYC_JSON" | jq "${filter}"
+ exit_fail "Instance ${id} did not receive a KYC access token"
+ fi
+ echo " DONE"
+}
+
+# Satisfy KYC measure M1 for the account behind access token $1 by
+# submitting the form that KYC-CHECK-C1 asks for.
+# taler-exchange-helper-measure-test-form then raises every limit to
+# TESTKUDOS:1000, which un-blocks the account.
+function submit_kyc_form() {
+ local token="$1"
+ local id
+ local n
+
+ # The measure is only instantiated when someone asks for it, so poll
+ # until the exchange offers something to do.
+ echo -n "Fetching KYC requirements ..."
+ for n in $(seq 1 100)
+ do
+ echo -n "."
+ curl -s -o kyc-info.json "${EXCHANGE_URL}kyc-info/${token}"
+ id=$(jq -r '.requirements[]? | select(.id != null) | .id' < kyc-info.json \
+ | head -n1)
+ if [ -n "$id" ]
+ then
+ break
+ fi
+ sleep 0.5
+ done
+ if [ -z "$id" ]
+ then
+ cat kyc-info.json
+ exit_fail "Exchange did not offer a KYC requirement to fulfill"
+ fi
+ echo " DONE (requirement $id)"
+
+ echo -n "Submitting the KYC form ..."
+ STATUS=$(curl -s -o kyc-upload.json -w "%{http_code}" \
+ -H "Content-Type: application/json" \
+ -X POST \
+ -d '{"FULL_NAME":"Merchant Fourtythree",
+ "DATE_OF_BIRTH":"2000-01-01",
+ "FORM_ID":"full_name_and_birthdate"}' \
+ "${EXCHANGE_URL}kyc-upload/${id}")
+ if [ "$STATUS" != "204" ]
+ then
+ cat kyc-upload.json
+ exit_fail "Expected 204 No Content from kyc-upload. Got: $STATUS"
+ fi
+ echo " DONE"
+}
+
+# Run the aggregator once, logging to aggregator-$1.log.
+#
+# Note the deliberate absence of '-y' (--kyc-off): that option makes the
+# aggregator wire funds out without consulting KYC at all, which is exactly
+# what this reference database must not do.
+#
+# The '-T' travels two hours forward so that the deposits' wire deadlines
+# have passed -- the wallet's integration-test order sets them a quarter of
+# an hour out, and simply moving them into the past instead makes the
+# aggregator spin forever on the KYC-blocked transient aggregation. The
+# price is that the wire_out rows come out stamped two hours ahead of when
+# the bank actually moved the money, which the auditor would rightly report
+# as a minor row inconsistency; the caller fixes the timestamps up again
+# once the transfer has been made.
+function run_aggregator() {
+ echo -n "Running exchange aggregator ($1) ..."
+ taler-exchange-aggregator \
+ -T 7200000000 \
+ -L INFO \
+ -t \
+ -c "${CONF}.edited" \
+ 2> "aggregator-$1.log" \
+ || exit_fail "aggregator failed"
+ echo " DONE"
+}
+
+# Pay TESTKUDOS:4 to the merchant instance $1, withdrawing TESTKUDOS:10
+# first. Uses its own wallet database so that coin selection does not
+# depend on what an earlier instance already spent.
+function pay_instance() {
+ local id="$1"
+ local base
+ local db="wallet-${id}.wdb"
+ local wlog="taler-wallet-cli-${id}.log"
+
+ if [ "$id" = "admin" ]
+ then
+ base="${MERCHANT_URL}"
+ else
+ base="${MERCHANT_URL}instances/${id}/"
+ fi
+
+ rm -f "$db"
+ echo -n "Running wallet against instance ${id} ..."
+ taler-wallet-cli \
+ --no-throttle \
+ --wallet-db="$db" \
+ api \
+ --expect-success \
+ 'runIntegrationTest' \
+ "$(jq -n '
+ {
+ amountToSpend: "TESTKUDOS:4",
+ amountToWithdraw: "TESTKUDOS:10",
+ corebankApiBaseUrl: $BANK_URL,
+ exchangeBaseUrl: $EXCHANGE_URL,
+ merchantBaseUrl: $MERCHANT_URL,
+ }' \
+ --arg MERCHANT_URL "$base" \
+ --arg EXCHANGE_URL "$EXCHANGE_URL" \
+ --arg BANK_URL "$BANK_URL"
+ )" &> "$wlog" || {
+ echo " FAILED! Last lines from $wlog:"
+ tail "$wlog"
+ exit 2
+ }
+ echo " DONE"
+ taler-wallet-cli --wallet-db="$db" run-until-done &>> "$wlog"
+}
+
+# ----------------------------------------------------------------------
+# Both instances prove that they own their bank account -- without that
+# the exchange refuses their deposits outright -- and are then paid the
+# same TESTKUDOS:4. Both amounts exceed the AGGREGATE threshold, so the
+# exchange accepts the coins but may not wire either of them out yet.
+# ----------------------------------------------------------------------
+
+setup_instance "admin" "$CLEARED_PAYTO" "Merchant43"
+kycauth_instance "admin"
+CLEARED_ACCESS_TOKEN="$ACCESS_TOKEN"
+
+setup_instance "blocked" "$BLOCKED_PAYTO" "Blocked44"
+kycauth_instance "blocked"
+BLOCKED_ACCESS_TOKEN="$ACCESS_TOKEN"
+
+pay_instance "admin"
+pay_instance "blocked"
+
+# The KYC rule only fires once the exchange actually tries to aggregate,
+# so run the aggregator before there is a measure to satisfy.
+run_aggregator "first"
+
+echo -n "Checking that no wire transfer happened yet ..."
+WIRE_OUT=$(psql -Aqt "auditor-basedb" \
+ -c "SELECT COUNT(*) FROM exchange.wire_out;")
+if [ "$WIRE_OUT" != "0" ]
+then
+ exit_fail "KYC should have blocked every aggregation, but got ${WIRE_OUT} wire_out rows"
+fi
+echo " DONE"
+
+# ----------------------------------------------------------------------
+# Only instance 'admin' passes the form. After that its funds may be
+# wired out; instance 'blocked' stays blocked, and the payout it is owed
+# stays parked in aggregation_transient for good.
+# ----------------------------------------------------------------------
+
+submit_kyc_form "$CLEARED_ACCESS_TOKEN"
+
+run_aggregator "second"
+
+echo -n "Running exchange transfer ..."
+taler-exchange-transfer \
+ -L INFO \
+ -t \
+ -c "${CONF}.edited" \
+ 2> transfer.log \
+ || exit_fail "transfer failed"
+echo " DONE"
+
+
+# Undo the side effect of the aggregator's time travel: the exchange
+# recorded the wire transfer as having been executed two hours from now,
+# while the bank stamped it with the real time. The auditor compares the
+# two and reports any disagreement of more than 15 minutes, so bring the
+# exchange's timestamp back to reality. wire_out.execution_date carries no
+# signature; test-auditor.sh's test 17 edits the very same field. Round to
+# whole seconds: a GNUNET_TIME_Timestamp may not carry a sub-second part,
+# and the helpers assert on that when they read the row back.
+echo -n "Correcting the wire_out execution dates ..."
+psql -Aqt "auditor-basedb" \
+ -c "UPDATE exchange.wire_out SET execution_date=1000000*((SELECT transaction_date FROM libeufin_bank.bank_account_transactions WHERE direction='debit' ORDER BY bank_transaction_id DESC LIMIT 1)/1000000);" \
+ > /dev/null
+echo " DONE"
+
+echo -n "Checking that exactly the cleared account was paid ..."
+WIRE_OUT=$(psql -Aqt "auditor-basedb" \
+ -c "SELECT COUNT(*) FROM exchange.wire_out;")
+if [ "$WIRE_OUT" != "1" ]
+then
+ exit_fail "Expected exactly one wire transfer, got ${WIRE_OUT}"
+fi
+echo " DONE"
+
+echo "KYC access token of the blocked account: ${BLOCKED_ACCESS_TOKEN}"
+
+# Dump database
+mkdir -p "$(dirname "$BASEDB")"
+
+echo "Dumping database ${BASEDB}.sql"
+pg_dump -O "auditor-basedb" | sed -e '/AS integer/d' > "${BASEDB}.sql"
+cp "${CONF}.edited" "${BASEDB}.conf"
+cp "$(taler-exchange-config -c "${CONF}.edited" -s exchange-offline -o MASTER_PRIV_FILE -f)" "${BASEDB}.mpriv"
+
+# clean up
+echo -n "Final clean up ..."
+kill -TERM "$SETUP_PID"
+wait
+unset SETUP_PID
+dropdb "auditor-basedb"
+echo " DONE"
+
+echo "====================================="
+echo "Finished generation of ${BASEDB}.sql"
+echo "====================================="
+
+exit 0
