diff options
Diffstat (limited to 'nlnet/task5')
| -rw-r--r-- | nlnet/task5/date-range/Dockerfile | 15 | ||||
| -rw-r--r-- | nlnet/task5/date-range/start-libeufin.sh | 35 | ||||
| -rwxr-xr-x | nlnet/task5/date-range/start.sh | 155 | ||||
| -rw-r--r-- | nlnet/task5/long-poll/Dockerfile | 14 | ||||
| -rwxr-xr-x | nlnet/task5/long-poll/start.sh | 134 | ||||
| -rw-r--r-- | nlnet/task5/performance/Dockerfile | 70 | ||||
| -rwxr-xr-x | nlnet/task5/performance/start.sh | 107 |
7 files changed, 530 insertions, 0 deletions
diff --git a/nlnet/task5/date-range/Dockerfile b/nlnet/task5/date-range/Dockerfile new file mode 100644 index 0000000..8d1224f --- /dev/null +++ b/nlnet/task5/date-range/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:stable + +RUN apt-get update +RUN apt-get install -y openjdk-17-jre git python3-pip curl jq sqlite3 postgresql python3-requests python3-click sudo + +# Installation +RUN git clone git://git.taler.net/libeufin +WORKDIR /libeufin +RUN git fetch && git checkout a614d433a8307468f1074114086ae0a47b848472 +RUN ./bootstrap +RUN ./configure --prefix=/usr/local +RUN make install +COPY start-libeufin.sh / +COPY start.sh / +ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task5/date-range/start-libeufin.sh b/nlnet/task5/date-range/start-libeufin.sh new file mode 100644 index 0000000..8f000a4 --- /dev/null +++ b/nlnet/task5/date-range/start-libeufin.sh @@ -0,0 +1,35 @@ +DB_CONN="postgresql:///libeufincheck" +export LIBEUFIN_SANDBOX_DB_CONNECTION=$DB_CONN +export LIBEUFIN_NEXUS_DB_CONNECTION=$DB_CONN + +echo -n Delete previous data... +libeufin-sandbox reset-tables +libeufin-nexus reset-tables +echo DONE +echo -n Configure the default demobank with MANA... +libeufin-sandbox config --with-signup-bonus --currency MANA default +echo DONE +echo -n Setting the default exchange at Sandbox... +libeufin-sandbox \ + default-exchange \ + "https://exchange.example.com/" \ + "payto://iban/NOTUSED" +echo DONE +echo -n Start the bank... +export LIBEUFIN_SANDBOX_ADMIN_PASSWORD=foo +libeufin-sandbox serve > sandbox.log 2>&1 & +SANDBOX_PID=$! +echo DONE +echo -n Wait for the bank... +curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5000/ &> /dev/null +echo DONE +echo -n Make one superuser at Nexus... +libeufin-nexus superuser test-user --password x +echo DONE +echo -n Launching Nexus... +libeufin-nexus serve &> nexus.log & +NEXUS_PID=$! +echo DONE +echo -n Waiting for Nexus... +curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5001/ &> /dev/null +echo DONE diff --git a/nlnet/task5/date-range/start.sh b/nlnet/task5/date-range/start.sh new file mode 100755 index 0000000..c61cfee --- /dev/null +++ b/nlnet/task5/date-range/start.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +# This script shows how Nexus can request histories from +# a particular time frame. Such request must succeed via +# two connection types: EBICS and x-libeufin-bank. EBICS +# ensures the fetching of fiat payments made to the regional +# currency authority, whereas x-libeufin-bank does it for +# the regional currency circuit. Note: the time-framed +# request is exceptional: it's used only after a complaint +# from a user where they didn't get their funds as expected. + +set -eu + +service postgresql start +sudo -u postgres createuser -s root +createdb libeufincheck + +echo -n Launching and confirguring LibEuFin.. +source /start-libeufin.sh &> /dev/null +# Register the Sandbox account. +export LIBEUFIN_SANDBOX_USERNAME=sandbox-user +export LIBEUFIN_SANDBOX_PASSWORD=foo +libeufin-cli \ + sandbox --sandbox-url http://localhost:5000/ \ + demobank \ + register +# x-libeufin-bank connection. +# Creating the x-libeufin-bank connection at Nexus. +export LIBEUFIN_NEXUS_USERNAME=test-user +export LIBEUFIN_NEXUS_PASSWORD=x +export LIBEUFIN_NEXUS_URL=http://localhost:5001 +libeufin-cli connections new-xlibeufinbank-connection \ + --bank-url "http://localhost:5000/demobanks/default/access-api" \ + --username sandbox-user \ + --password foo \ + xlibeufinbankconn +# Connecting the x-libeufin-bank connection... +libeufin-cli connections connect xlibeufinbankconn +# Importing the bank account under a local name at Nexus. +# Importing the x-libeufin-bank account locally.. +libeufin-cli connections import-bank-account \ + --offered-account-id sandbox-user \ + --nexus-bank-account-id foo-at-nexus xlibeufinbankconn + +# EBICS connection. +## Sandbox side. +export LIBEUFIN_SANDBOX_USERNAME=admin +# "Create EBICS host at Sandbox..." +libeufin-cli sandbox \ + --sandbox-url http://localhost:5000 \ + ebicshost create --host-id wwwebics +# Create nlnet EBICS subscriber at Sandbox +libeufin-cli sandbox \ + --sandbox-url http://localhost:5000 \ + demobank new-ebicssubscriber --host-id wwwebics \ + --user-id nlnet --partner-id nlnet \ + --bank-account sandbox-user # that's a username _and_ a bank account name +## Nexus side. +export LIBEUFIN_NEXUS_USERNAME=test-user +export LIBEUFIN_NEXUS_PASSWORD=x +export LIBEUFIN_NEXUS_URL=http://localhost:5001 +# Creating the EBICS connection at Nexus... +libeufin-cli connections new-ebics-connection \ + --ebics-url "http://localhost:5000/ebicsweb" \ + --host-id wwwebics \ + --partner-id nlnet \ + --ebics-user-id nlnet \ + ebicsconn +# Setup EBICS keying... +libeufin-cli connections connect ebicsconn > /dev/null +# Download bank account name from Sandbox... +libeufin-cli connections download-bank-accounts ebicsconn +# Importing bank account info into Nexus... +libeufin-cli connections import-bank-account \ + --offered-account-id sandbox-user \ + --nexus-bank-account-id bar-at-nexus ebicsconn +echo DONE + +FIRST_JAN_2020="1577833200000" # in milliseconds +END_DEC_2019="2019-12-30" +MID_JAN_2020="2020-01-15" + +echo Make sample transaction.. +# 0, setup and start services. +libeufin-sandbox make-transaction \ + --credit-account=admin \ + --debit-account=sandbox-user MANA:2 \ + "task5" # subject. +echo DONE + +echo -n Articifially set the transaction date to $FIRST_JAN_2020.. +# 1, set artificial time for the transaction at January, 1st 2020. +echo "UPDATE bankaccounttransactions SET date='$FIRST_JAN_2020' WHERE subject='task5'" | psql -q -d libeufincheck +echo DONE + +# 2, retrieve the transaction via Nexus, for both +# connections and by asking for a (narrow) time frame +# that includes the 2020-01-01 payment. + +echo -n Nexus: syncing banking records for the time frame $END_DEC_2019-$MID_JAN_2020 via EBICS.. +# Fetch time-framed payments via EBICS. +libeufin-cli \ + accounts \ + fetch-transactions \ + --level=report \ + --range-type=time-range \ + --start=$END_DEC_2019 \ + --end=$MID_JAN_2020 \ + bar-at-nexus > /dev/null # EBICS +echo DONE + +echo Showing the synced data.. +# Now checks if Nexus ingested and shows the +# expected payment. +libeufin-cli \ + accounts \ + transactions \ + bar-at-nexus +echo DONE + +echo Resetting the Nexus database.. +# Bring the database state so that Nexus does not hold any payment. +echo "DELETE FROM nexusbanktransactions" | psql -d libeufincheck +echo "DELETE FROM nexusbankmessages" | psql -d libeufincheck +echo DONE + +echo Checking that no payment data appears after the reset.. +# Double-checking that the future steps start +# without the previous transactions. +libeufin-cli \ + accounts \ + transactions \ + foo-at-nexus # FIXME: put a 204 No Content check? +echo DONE + +# Fetch time-framed payments via x-libeufin-bank. +echo Nexus: syncing banking records for the time frame ${END_DEC_2019}_${MID_JAN_2020} via x-libeufin-bank.. +libeufin-cli \ + accounts \ + fetch-transactions \ + --level=statement \ + --range-type=time-range \ + --start=$END_DEC_2019 \ + --end=$MID_JAN_2020 \ + foo-at-nexus +echo DONE + +# As in the previous case, now Nexus should show +# the 2020-01-01 the payment. +echo Showing the synced data.. +libeufin-cli \ + accounts \ + transactions \ + foo-at-nexus # FIXME: put a 200 OK check? +echo DONE diff --git a/nlnet/task5/long-poll/Dockerfile b/nlnet/task5/long-poll/Dockerfile new file mode 100644 index 0000000..713e47e --- /dev/null +++ b/nlnet/task5/long-poll/Dockerfile @@ -0,0 +1,14 @@ +FROM debian:stable + +RUN apt-get update +RUN apt-get install -y openjdk-17-jre git python3-pip curl jq sqlite3 postgresql python3-requests python3-click sudo libgnunet0.19 + +# Installation +RUN git clone git://git.taler.net/libeufin +WORKDIR /libeufin +RUN git fetch && git checkout 934a73b09b9e9abba348e15ddc058df5bb9cd6a3 +RUN ./bootstrap +RUN ./configure --prefix=/usr/local +RUN make install +COPY start.sh / +ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task5/long-poll/start.sh b/nlnet/task5/long-poll/start.sh new file mode 100755 index 0000000..46a0af2 --- /dev/null +++ b/nlnet/task5/long-poll/start.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +set -eu + +service postgresql start +sudo -u postgres createuser -s root +createdb libeufincheck + +wire_transfer () { + RESERVE_PUB=$(gnunet-ecc -g1 /tmp/www &> /dev/null && gnunet-ecc -p /tmp/www) + DB_CONN="postgresql:///libeufincheck" + libeufin-sandbox \ + make-transaction \ + --credit-account=sandbox-user \ + --debit-account=admin MANA:2 \ + $RESERVE_PUB +} + +WITH_TASKS=1 +echo RUNNING SANDBOX-NEXUS EBICS PAIR +jq --version &> /dev/null || (echo "'jq' command not found"; exit 77) +curl --version &> /dev/null || (echo "'curl' command not found"; exit 77) + +DB_CONN="postgresql:///libeufincheck" +export LIBEUFIN_SANDBOX_DB_CONNECTION=$DB_CONN +export LIBEUFIN_NEXUS_DB_CONNECTION=$DB_CONN + +echo -n Delete previous data... +libeufin-sandbox reset-tables +libeufin-nexus reset-tables +echo DONE +echo -n Configure the default demobank with MANA... +libeufin-sandbox config --with-signup-bonus --currency MANA default +echo DONE +echo -n Setting the default exchange at Sandbox... +libeufin-sandbox \ + default-exchange \ + "https://exchange.example.com/" \ + "payto://iban/NOTUSED" +echo DONE +echo -n Start the bank... +export LIBEUFIN_SANDBOX_ADMIN_PASSWORD=foo +libeufin-sandbox serve > sandbox.log 2>&1 & +SANDBOX_PID=$! +echo DONE +echo -n Wait for the bank... +curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5000/ &> /dev/null +echo DONE +echo -n Make one superuser at Nexus... +libeufin-nexus superuser test-user --password x +echo DONE +echo -n Launching Nexus... +libeufin-nexus serve &> nexus.log & +NEXUS_PID=$! +echo DONE +echo -n Waiting for Nexus... +curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5001/ &> /dev/null +echo DONE + +echo -n "Register the Sandbox account..." +export LIBEUFIN_SANDBOX_USERNAME=sandbox-user +export LIBEUFIN_SANDBOX_PASSWORD=foo +libeufin-cli \ + sandbox --sandbox-url http://localhost:5000/ \ + demobank \ + register +echo DONE +echo -n Creating the x-libeufin-bank connection at Nexus... +export LIBEUFIN_NEXUS_USERNAME=test-user +export LIBEUFIN_NEXUS_PASSWORD=x +export LIBEUFIN_NEXUS_URL=http://localhost:5001 +# echoing the password to STDIN, as that is a "prompt" option. +libeufin-cli connections new-xlibeufinbank-connection \ + --bank-url "http://localhost:5000/demobanks/default/access-api" \ + --username sandbox-user \ + --password foo \ + wwwconn +echo DONE +echo -n Connecting the x-libeufin-bank connection... +libeufin-cli connections connect wwwconn +echo DONE +# Importing the bank account under a local name at Nexus. +echo -n Importing the x-libeufin-bank account locally.. +libeufin-cli connections import-bank-account \ + --offered-account-id sandbox-user \ + --nexus-bank-account-id foo-at-nexus wwwconn +echo DONE +echo -n Create the Taler facade at Nexus... +libeufin-cli facades \ + new-taler-wire-gateway-facade \ + --currency TESTKUDOS --facade-name test-facade \ + wwwconn foo-at-nexus +echo DONE +if test 1 = $WITH_TASKS; then + echo -n Creating submit transactions task.. + libeufin-cli accounts task-schedule \ + --task-type submit \ + --task-name www-payments \ + --task-cronspec "* * *" \ + foo-at-nexus || true + # Tries every second. Ask C52 + echo DONE + echo -n Creating fetch transactions task.. + # Not idempotent, FIXME #7739 + libeufin-cli accounts task-schedule \ + --task-type fetch \ + --task-name www-history \ + --task-cronspec "* * *" \ + --task-param-level statement \ + --task-param-range-type since-last \ + foo-at-nexus || true + echo DONE +else + echo NOT creating background tasks! +fi + +echo +echo Services are online! The following shell offers a 'wire_transfer' +echo command that wires money to Nexus 'test-user'. Give it after having +echo connected an HTTP client that long-polls to Nexus. As an example, a +echo 100 seconds long-poller to Nexus is the following command: +echo curl -v -u test-user:x "'http://localhost:5001/facades/test-facade/taler-wire-gateway/history/incoming?delta=5&long_poll_ms=100000'" +echo +echo Hint: after having issued the previous command and having observed +echo that it actually long-polls, press CTRL-Z to send it in the background, +echo "then wire the funds to the long-poller with 'wire_transfer'," +echo "and finally give 'fg 1' to bring the long-poller in the foreground." +echo If the client now shows a response, then the long-polling mechanism +echo worked. +echo + +cd / +export -f wire_transfer +bash diff --git a/nlnet/task5/performance/Dockerfile b/nlnet/task5/performance/Dockerfile new file mode 100644 index 0000000..4daeaf0 --- /dev/null +++ b/nlnet/task5/performance/Dockerfile @@ -0,0 +1,70 @@ +FROM debian:stable + +RUN apt-get update +RUN apt-get install -y \ + git \ + openjdk-17-jre \ + python3-pip \ + curl \ + jq \ + postgresql \ + python3-requests \ + python3-click \ + sudo \ + time \ + autoconf \ + autopoint \ + libtool \ + texinfo \ + libgcrypt-dev \ + libidn11-dev \ + zlib1g-dev \ + libunistring-dev \ + libjansson-dev \ + recutils \ + libsqlite3-dev \ + libpq-dev \ + libcurl4-openssl-dev \ + libsodium-dev \ + libqrencode-dev \ + zip + +# Installation +RUN git clone git://git.taler.net/libeufin +WORKDIR /libeufin +RUN git fetch && git checkout 4bc5f38f571a45d427f73813ec3846bf59413afa +RUN ./bootstrap +RUN ./configure --prefix=/usr/local +RUN make install +WORKDIR / +RUN git clone git://git.gnunet.org/libmicrohttpd +WORKDIR /libmicrohttpd +RUN ./bootstrap +RUN ./configure --disable-doc +RUN make install +WORKDIR / +RUN git clone git://git.gnunet.org/gnunet +WORKDIR /gnunet +RUN apt-get install -y python3-sphinx python3-sphinx-rtd-theme # Move up? +RUN ./bootstrap +RUN ./configure +RUN pip3 install --break-system-packages htmlark +RUN make install +WORKDIR / +RUN git clone git://git.taler.net/exchange +WORKDIR /exchange +RUN ./bootstrap +RUN ./configure +RUN make install +WORKDIR / +RUN git clone git://git.taler.net/merchant +WORKDIR /merchant +RUN ./bootstrap +RUN ./configure +RUN make install +WORKDIR / + +COPY start.sh / +RUN apt-get install -y wget +RUN apt-get install -y bc +ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task5/performance/start.sh b/nlnet/task5/performance/start.sh new file mode 100755 index 0000000..2cc9175 --- /dev/null +++ b/nlnet/task5/performance/start.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# This script shows, via runnuing the benchmark, how +# the LibEuFin database connections are significantly +# shorter than the benchmark total time. + +# For this reason, it can only be that LibEuFin opens +# and closes many PostgreSQL connections, as it is required +# by milestone #3. + +set -eu + +export HOW_MANY_WITHDRAWALS=100 + +service postgresql start +sudo -u postgres createuser -s root + +# Activating the disconnection logs. +sudo -u postgres psql -q -c "ALTER SYSTEM SET log_disconnections = 'on'" -c "SELECT pg_reload_conf()" > /dev/null + +# Converts AA:BB:CC.DDD to milliseconds. +convert_pg_time_to_ms () { + awk -F[.:] '{SECS=(60*60*$1)+(60*$2)+$3; MILLI=$4; TOTAL_MS=(SECS*1000)+MILLI; print TOTAL_MS}' +} + +createdb talercheck +export LD_LIBRARY_PATH=/usr/local/lib + +prepare_and_run () { + taler-unified-setup.sh \ + -Wwemtns \ + -c /exchange/src/benchmark/benchmark-cs.conf \ + -u exchange-account-2 &> /check_ready.txt & + # Wait that the prep. went through. + echo -n Waiting the unified setup to complete.. + READY="NO" + for i in `seq 100` true; do + if grep -q "<<READY>>" /check_ready.txt; then + READY="YES" + break + fi + echo -n "."; sleep 1 + done + + if test $READY = "YES"; then + echo "DONE" + else + cat /check_ready.txt + echo FAIL + exit 1 + fi + + echo Running the benchmark.. + taler-exchange-benchmark \ + -c /exchange/src/benchmark/benchmark-cs.conf.edited \ + -u exchange-account-2 \ + -L WARNING \ + -n 1 \ + -r $HOW_MANY_WITHDRAWALS +} + +export -f prepare_and_run +/usr/bin/time -o /benchmark-wall-clock-time.txt --format=%e bash -c "prepare_and_run" + +NEXUS_PID=$(cat /libeufin-nexus.pid) +SANDBOX_PID=$(cat /libeufin-sandbox.pid) + +if test -z $NEXUS_PID; then + echo Could not find Nexus PID, failing. + exit 1 +fi + +if test -z $SANDBOX_PID; then + echo Could not find Sandbox PID, failing. + exit 1 +fi + +# Convert the wall clock time to milliseconds, to make +# it compatible with the format as GREPped through Postgres logs. +BENCHMARK_TOT_MS=$(awk -F. '{t=($1 * 1000 + $2 * 10)} END {print t}' /benchmark-wall-clock-time.txt) + +NEXUS_LONGEST_DB_SESSION_MS=$(grep disconnection < /var/log/postgresql/postgresql-15-main.log | grep $NEXUS_PID | grep -o "session time:.*$" | grep -o [0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9] | convert_pg_time_to_ms | sort -n | tail -n 1) + +SANDBOX_LONGEST_DB_SESSION_MS=$(grep disconnection < /var/log/postgresql/postgresql-15-main.log | grep $SANDBOX_PID | grep -o "session time:.*$" | grep -o [0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9] | convert_pg_time_to_ms | sort -n | tail -n 1) + +if test $NEXUS_LONGEST_DB_SESSION_MS -gt $BENCHMARK_TOT_MS; then + echo Nexus had a DB session longer than the benchmark itself, failing. + exit 1 +fi + +if test $SANDBOX_LONGEST_DB_SESSION_MS -gt $BENCHMARK_TOT_MS; then + echo Sandbox had a DB session longer than the benchmark itself, failing. + exit 1 +fi + +NEXUS_TIME_PORTION=$(echo "($NEXUS_LONGEST_DB_SESSION_MS / $BENCHMARK_TOT_MS) * 100" | bc -lq | sed 's/^\./0./') +SANDBOX_TIME_PORTION=$(echo "($SANDBOX_LONGEST_DB_SESSION_MS / $BENCHMARK_TOT_MS) * 100" | bc -lq | sed 's/^\./0./') + +# Here: the further from 1 the better. +echo Nexus longest DB session is $NEXUS_TIME_PORTION percent of the total benchmark time. +echo Sandbox longest DB session is $SANDBOX_TIME_PORTION percent of the total benchmark time. + +# Now show the total space occupied by the database. +# Although that's a _total_ estimate, it'll anyhow show +# that _also_ libeufin has reasonable data usage. +TOTAL_DB_SPACE=$(echo "SELECT pg_size_pretty(pg_database_size('talercheck'))" | psql -d talercheck | grep "^ [0-9]" | tr -d "[:blank:]") +echo "The total space occupied by the database for $HOW_MANY_WITHDRAWALS withdrawals is $TOTAL_DB_SPACE" |