test-sync.sh (9251B)
1 #!/usr/bin/env bash 2 # 3 # This file is part of TALER 4 # Copyright (C) 2014-2023 Taler Systems SA 5 # 6 # TALER is free software; you can redistribute it and/or modify it under the 7 # terms of the GNU General Public License as published by the Free Software 8 # Foundation; either version 3, or (at your option) any later version. 9 # 10 # TALER is distributed in the hope that it will be useful, but WITHOUT ANY 11 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 12 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 13 # 14 # You should have received a copy of the GNU General Public License along with 15 # TALER; see the file COPYING. If not, If not, see <http://www.gnu.org/license> 16 # 17 # shellcheck disable=SC2317 18 19 set -eu 20 21 # Exit, with status code "skip" (no 'real' failure) 22 function exit_skip() { 23 echo "SKIPPING test: $1" 24 exit 77 25 } 26 27 # Exit, with error message (hard failure) 28 function exit_fail() { 29 echo "FAILING test: $1" 30 exit 1 31 } 32 33 # Cleanup to run whenever we exit 34 function cleanup() { 35 if [ -n "${POSTGRES_PATH:-}" ] 36 then 37 "${POSTGRES_PATH}/pg_ctl" -D "$TMPDIR" stop &> /dev/null || true 38 fi 39 for n in $(jobs -p) 40 do 41 kill "$n" 2> /dev/null || true 42 done 43 wait 44 } 45 46 # Install cleanup handler (except for kill -9) 47 trap cleanup EXIT 48 49 # If this script is not run as root, 50 # create the temporary storage space for postgres. 51 # Will set PGHOST accordingly 52 function perform_initdb() { 53 # Available directly in path? 54 INITDB_BIN=$(command -v initdb) || true 55 if [[ -n "$INITDB_BIN" ]]; then 56 echo " FOUND (in path) at $INITDB_BIN" 57 else 58 HAVE_INITDB=$(find /usr -name "initdb" | head -1 2> /dev/null | grep postgres) || exit_skip " MISSING" 59 echo " FOUND at " "$(dirname "$HAVE_INITDB")" 60 INITDB_BIN=$(echo "$HAVE_INITDB" | grep bin/initdb | grep postgres | sort -n | tail -n1) 61 fi 62 echo -n "Setting up Postgres DB" 63 POSTGRES_PATH=$(dirname "$INITDB_BIN") 64 TMPDIR="$MYDIR/postgres/" 65 mkdir -p "$TMPDIR" 66 "$INITDB_BIN" --no-sync --auth=trust -D "${TMPDIR}" \ 67 > "${MYDIR}/postgres-dbinit.log" \ 68 2> "${MYDIR}/postgres-dbinit.err" 69 echo " DONE" 70 mkdir "${TMPDIR}/sockets" 71 echo -n "Launching Postgres service" 72 cat - >> "$TMPDIR/postgresql.conf" <<EOF 73 unix_socket_directories='${TMPDIR}/sockets' 74 fsync=off 75 max_wal_senders=0 76 synchronous_commit=off 77 wal_level=minimal 78 listen_addresses='' 79 EOF 80 grep -v host \ 81 < "$TMPDIR/pg_hba.conf" \ 82 > "$TMPDIR/pg_hba.conf.new" 83 mv "$TMPDIR/pg_hba.conf.new" "$TMPDIR/pg_hba.conf" 84 "${POSTGRES_PATH}/pg_ctl" \ 85 -D "$TMPDIR" \ 86 -l /dev/null \ 87 start \ 88 > "${MYDIR}/postgres-start.log" \ 89 2> "${MYDIR}/postgres-start.err" 90 echo " DONE" 91 PGHOST="$TMPDIR/sockets" 92 export PGHOST 93 } 94 95 # Exercise an upgrade from a replica which copied neither transfer keys 96 # nor the reveal flag. The plain reference database contains real melts. 97 function check_refresh_sync() 98 { 99 local fields="SELECT rc,rc_version,transfer_pubs,revealed FROM exchange.refresh ORDER BY rc" 100 local expected actual revealed 101 psql -Xq -v ON_ERROR_STOP=1 talercheck-in -c "UPDATE exchange.refresh SET revealed=FALSE" 102 expected=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-in -c "$fields") 103 [[ -n "$expected" ]] || exit_fail "Missing refresh fixture" 104 105 psql -Xq -v ON_ERROR_STOP=1 talercheck-out -c \ 106 "UPDATE exchange.refresh SET rc_version=0,transfer_pubs=NULL,revealed=FALSE" 107 actual=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-out -c "$fields") 108 [[ "$actual" != "$expected" ]] || exit_fail "Missing refresh with transfer keys" 109 110 taler-auditor-sync -s test-sync-in.conf -d test-sync-out.conf -t -b 1 111 actual=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-out -c "$fields") 112 [[ "$actual" == "$expected" ]] || exit_fail "Refresh metadata differs after sync" 113 114 psql -Xq -v ON_ERROR_STOP=1 talercheck-in -c "UPDATE exchange.refresh SET revealed=TRUE" 115 expected=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-in -c "$fields") 116 taler-auditor-sync -s test-sync-in.conf -d test-sync-out.conf -t -b 1 117 actual=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-out -c "$fields") 118 [[ "$actual" == "$expected" ]] || exit_fail "Later reveal missing on replica" 119 120 # An old writer running after migration gets the current version by 121 # default, even though it omits transfer_pubs from its INSERT. 122 psql -Xq -v ON_ERROR_STOP=1 talercheck-in -c \ 123 "UPDATE exchange.refresh SET rc_version=DEFAULT WHERE transfer_pubs IS NOT NULL" 124 psql -Xq -v ON_ERROR_STOP=1 talercheck-out -c \ 125 "UPDATE exchange.refresh SET rc_version=DEFAULT WHERE transfer_pubs IS NOT NULL" 126 expected=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-in -c "$fields") 127 for revealed in FALSE TRUE; do 128 # Also exercise a replica which already copied the reveal flag. 129 psql -Xq -v ON_ERROR_STOP=1 talercheck-out -c \ 130 "UPDATE exchange.refresh SET transfer_pubs=NULL,revealed=$revealed WHERE rc_version=2" 131 actual=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-out -c "$fields") 132 [[ "$actual" != "$expected" ]] || exit_fail "Missing current-version refresh fixture" 133 taler-auditor-sync -s test-sync-in.conf -d test-sync-out.conf -t -b 1 134 actual=$(psql -XAt -v ON_ERROR_STOP=1 talercheck-out -c "$fields") 135 [[ "$actual" == "$expected" ]] || exit_fail "Missing transfer keys after rolling upgrade" 136 done 137 } 138 139 function check_with_database() 140 { 141 echo -n "Testing synchronization logic ..." 142 143 dropdb talercheck-in 2> /dev/null || true 144 dropdb talercheck-out 2> /dev/null || true 145 146 createdb talercheck-in || exit 77 147 createdb talercheck-out || exit 77 148 echo -n "." 149 150 taler-exchange-dbinit -c test-sync-out.conf 151 echo -n "." 152 psql -Aqt talercheck-in \ 153 -q -1 \ 154 -f "$1.sql" \ 155 >/dev/null \ 156 || exit_skip "Failed to load database" 157 158 echo -n "." 159 taler-auditor-sync \ 160 -s test-sync-in.conf \ 161 -d test-sync-out.conf -t 162 163 # cs_nonce_locks excluded: no point 164 for table in denominations denomination_revocations kyc_targets wire_targets reserves reserves_in reserves_close reserves_open_requests reserves_open_deposits auditors auditor_denom_sigs exchange_sign_keys signkey_revocations known_coins refresh batch_deposits coin_deposits refunds wire_out aggregation_tracking aggregation_deferrals wire_fee global_fee recoup recoup_refresh extensions policy_details policy_fulfillments purse_requests purse_decision purse_merges purse_deposits account_merges history_requests close_requests wads_out wad_out_entries wads_in wad_in_entries profit_drains aml_staff purse_deletion withdraw legitimization_measures legitimization_outcomes legitimization_processes kyc_attributes aml_history kyc_events kycauths_in 165 do 166 echo -n "." 167 CIN=$(echo "SELECT COUNT(*) FROM exchange.$table" | psql talercheck-in -Aqt) 168 COUT=$(echo "SELECT COUNT(*) FROM exchange.$table" | psql talercheck-out -Aqt) 169 170 if [ "${CIN}" != "${COUT}" ] 171 then 172 dropdb talercheck-in 173 dropdb talercheck-out 174 echo "FAIL" 175 exit_fail "Record count mismatch: $CIN / $COUT in table $table" 176 fi 177 done 178 179 if [[ "$1" == "$MYDIR/auditor-basedb" ]]; then 180 check_refresh_sync 181 fi 182 183 echo -n ". " 184 dropdb talercheck-in 185 dropdb talercheck-out 186 187 echo "PASS" 188 fail=0 189 } 190 191 # test required commands exist 192 echo "Testing for jq" 193 jq -h > /dev/null || exit_skip "jq required" 194 echo "Testing for taler-merchant-config" 195 taler-merchant-config -h > /dev/null || exit_skip "taler-merchant-config required" 196 echo "Testing for taler-merchant-httpd" 197 taler-merchant-httpd -h > /dev/null || exit_skip "taler-merchant-httpd required" 198 echo "Testing for faketime" 199 faketime -h > /dev/null || exit_skip "faketime required" 200 echo "Testing for libeufin-bank" 201 libeufin-bank --help >/dev/null </dev/null 2> /dev/null || exit_skip "libeufin-bank required" 202 echo "Testing for taler-wallet-cli" 203 taler-wallet-cli -h >/dev/null </dev/null 2>/dev/null || exit_skip "taler-wallet-cli required" 204 205 MYDIR=$(mktemp -d /tmp/taler-auditor-basedbXXXXXX) 206 207 echo -n "Testing for Postgres ..." 208 [ $(id -u) == 0 ] || perform_initdb 209 210 echo "Generating fresh database at $MYDIR" 211 if faketime -f '-1 d' ./generate-auditor-basedb.sh -d "$MYDIR/auditor-basedb" 212 then 213 check_with_database "$MYDIR/auditor-basedb" 214 if [ x$fail != x0 ] 215 then 216 exit "$fail" 217 fi 218 else 219 echo "Generation failed" 220 exit 77 221 fi 222 223 # The plain reference database has no KYC activity at all, so every 224 # KYC-related table in it is empty and replicating them is vacuously 225 # correct. Run the whole thing again against the KYC reference database, 226 # which does populate kycauths_in, kyc_attributes and the 227 # legitimization_* tables. 228 echo "Generating fresh KYC database at $MYDIR" 229 if faketime -f '-1 d' ./generate-kyc-basedb.sh -d "$MYDIR/kyc-basedb" 230 then 231 check_with_database "$MYDIR/kyc-basedb" 232 if [ x$fail != x0 ] 233 then 234 exit "$fail" 235 fi 236 else 237 echo "Generation failed" 238 exit 77 239 fi 240 241 echo "Cleaning up $MYDIR..." 242 rm -rf "$MYDIR" || echo "Removing $MYDIR failed" 243 exit 0