commit fd40f3368c200edc37e2586866152f25152e89e4 parent 5fc4e6c5354cfe3affa7b2584080e18abdf147b3 Author: Christian Grothoff <christian@grothoff.org> Date: Sun, 12 Jan 2025 16:11:19 +0100 -more deployment git cleanups Diffstat:
97 files changed, 228 insertions(+), 1553 deletions(-)
diff --git a/README.md b/README.md @@ -13,24 +13,32 @@ Buildbot jobs are used to: (primarily demo.taler.net, test.taler.net, int.taler.net) - run unit tests -A namespace/process jail is used to ensure there are no port conflicts or -surviving processes for unit tests that require ports that might conflict. - Deployment Layout ================= +aptly/ -- FIXME +bootstrpa-docker/ -- FIXME buildbot/ -- scripts and configuration for the buildmaster and jobs launched by it -envcfg/ -- Git tags for various deployments, specify which versions should be used +docker/ -- FIXME +head.taler.net/ -- FIXME history/ -- directory for NOT MAINTAINED scripts +historic/depo/ -- configuration files for depolymerizer deployment historic/docker/ -- dockerized Taler deployment historic/guix/ -- Guix packages for Taler +historic/vagrant/ -- FIXME netjail/ -- scripts to create a namespaces to ensure buildslaves do not compete for resources +netzbon/ -- scripts to assist netzbon with their deployment postmortem/ -- postmortem reports about major incidents we have had with lessons learned private-keys/ -- deployment private keys so that keys do not change if we migrate elsewhere -sign-gnu-artefact -- shell script for making releases and preparing upload to ftp.gnu.org -systemd-services/ -- service definitions for buildbot build-slaves -taler-sitesbuild/ -- scripts for updating our Web sites +regional-currency/ -- scripts to setup a regional currency with minimum effort +sign-gnu-artefact/ -- shell script for making releases and preparing upload to ftp.gnu.org +splitops/ -- shell for multi-sig access control to lock down root on production systems +systemd-services/ -- service definitions for buildbot (master and slaves) +weblate/ -- FIXME +worker-codecoverage/ -- scripts for running code coverage (currently down!) +worker-linkchecker/ -- buildbot worker scripts to check for broken links +worker-sites/ -- buildbot worker scripts for updating our Web sites TODOS: diff --git a/buildbot/bootstrap-scripts/bootstrap-codespell b/buildbot/bootstrap-scripts/bootstrap-codespell @@ -1,51 +0,0 @@ -#!/bin/bash - -# Bootstrap the Taler setup for the user account that -# is currently logged in. - -# Generates a setup for a single user, -# including a postgresql DB. - -set -eu - -BRANCH=master -REPOS="libmicrohttpd gnunet" - -cd $HOME - -pip3 install --user codespell - -for component in $REPOS; do - if ! test -d $HOME/$component; then - git clone git://git.gnunet.org/$component.git - fi -done -for component in $REPOS; do - echo "Checking out $component to $BRANCH" - git -C $HOME/$component checkout $BRANCH -done - -REPOS="exchange merchant wallet-core sync bank twister" - -for component in $REPOS; do - if ! test -d $HOME/$component; then - git clone git://git.taler.net/$component.git - fi -done - -for component in $REPOS; do - echo "Checking out $component to $BRANCH" - git -C $HOME/$component checkout $BRANCH -done - -if [[ ! -d worker ]]; then - buildbot-worker create-worker --umask=0o22 ~/worker localhost:9989 codespell-worker codespell-pass -fi - -mkdir -p ~/.config/systemd/user/ -cp systemd-services/buildbot-worker-doc.service ~/.config/systemd/user/ - -systemctl --user daemon-reload || echo "Please use 'machinectl shell codespell@.host' to log in to use this script" - -systemctl --user enable buildbot-worker-doc.service -systemctl --user start buildbot-worker-doc.service diff --git a/buildbot/bootstrap-scripts/bootstrap-linkchecker b/buildbot/bootstrap-scripts/bootstrap-linkchecker @@ -1,37 +0,0 @@ -#!/bin/bash - -# Bootstrap the Taler setup for the user account that -# is currently logged in. - -# Generates a setup for a single user, -# including a postgresql DB. - -set -eu - -BRANCH=master -REPOS="deployment" - -cd $HOME - -for component in $REPOS; do - if ! test -d $HOME/$component; then - git clone git://git.taler.net/$component.git - fi -done - -for component in $REPOS; do - echo "Checking out $component to $BRANCH" - git -C $HOME/$component checkout $BRANCH -done - -if [[ ! -d worker ]]; then - buildbot-worker create-worker --umask=0o22 ~/worker 127.0.0.1 linkchecker-worker linkchecker-pass -fi - -#mkdir -p ~/.config/systemd/user/ -#cp systemd-services/buildbot-worker-doc.service ~/.config/systemd/user/ - -#systemctl --user daemon-reload || echo "Please use 'machinectl shell docbuilder@.host' to log in to use this script" - -#systemctl --user enable buildbot-worker-linkchecker.service -#systemctl --user start buildbot-worker-linkchecker.service diff --git a/buildbot/linkchecker.sh b/buildbot/linkchecker.sh @@ -1,44 +0,0 @@ -#!/bin/bash -#set -v -# Removed because wget errors with error 8 (Server issued an error response.) -#set -e - -## This script will scan www.taler.net for broken links and e-mail a copy of the log if any are found. - -logfile="$HOME/linkchecker.log" -wait_time="1" -recurse_level="1" -ignore_list="(.*)demo.taler.net(.*)\/orders\/(.*)" # appears to do *nothing* - -# Remove old log - -if [ -f "$logfile" ] - then - echo $logfile " exists. Moving to" $logfile".old." - mv $logfile $logfile.old - else - echo "Info: existing log file '$logfile' not found." -fi - -podman build -t linkchecker:latest -f "$HOME/deployment/buildbot/linkchecker.Containerfile" "$HOME/deployment/buildbot" - -# Use wget to scan hosts and save output -for url in "https://www.taler.net/" "https://docs.taler.net/" "https://taler-systems.com/" "https://demo.taler.net/" "https://bank.demo.taler.net/" "https://shop.demo.taler.net/" "https://donations.demo.taler.net/" ; do - echo -e "\n\n#############################\n## Starting check on ${url}\n#############################\n" - podman run --rm localhost/linkchecker:latest \ - linkchecker \ - --no-robots \ - --check-extern \ - --recursion-level="$recurse_level" \ - "$url" | tee --append "$logfile" -done - -# Search the log for the phrase "broken link" as this is what wget will report -if grep -Rl 'Error' $logfile - then - echo "Found broken links. Build should fail (exit 1), triggering e-mail notification." - exit 1 - else - echo "No broken links found. Nothing more to do." - exit 0 -fi diff --git a/buildbot/make-buildbot-containerworker.sh b/buildbot/make-buildbot-containerworker.sh @@ -3,13 +3,16 @@ # Bootstrap the Taler setup for the container-worker. set -eu -export PATH=/home/buildbot-master/.local/bin:$PATH if ! test "container-worker" = $(whoami); then echo Only run as the 'container-worker' user. exit 1 fi +export PATH=$HOME/.local/bin:$PATH + +pip install --user --break-system-packages buildbot-worker + mkdir -p workspace if [[ ! -d worker ]]; then diff --git a/buildbot/make-buildbot-linkchecker.sh b/buildbot/make-buildbot-linkchecker.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Bootstrap the Taler setup for the linkchecker-worker. + +set -eu + +if ! test "linkchecker-worker" = $(whoami); then + echo Only run as the 'linkchecker-worker' user. + exit 1 +fi + +cd + +export PATH=$HOME/.local/bin:$PATH + +pip install --user --break-system-packages buildbot-worker + +if ! test -d "$HOME/deployment"; +then + git clone git://git.taler.net/deployment.git +fi + +if [[ ! -d worker ]]; then + buildbot-worker create-worker --umask=0o22 ~/worker 127.0.0.1:9988 linkchecker-worker linkchecker-pass +fi diff --git a/buildbot/make-buildbot-master.sh b/buildbot/make-buildbot-master.sh @@ -11,29 +11,36 @@ fi cd +export PATH=$HOME/.local/bin:$PATH + +pip install --user --break-system-packages buildbot buildbot-www buildbot-profiler buildbot-prometheus buildbot-console-view buildbot-www-react buildbot-grid-view buildbot-enhanced-ui buildbot-react-waterfall-view buildbot-worker buildbot-waterfall-view + BRANCH=master REPOS="deployment" -cd $HOME - -for component in $REPOS; do - if ! test -d $HOME/$component; then - git clone git://git.taler.net/$component.git - fi +for component in $REPOS; +do + if ! test -d $HOME/$component; + then + git clone git://git.taler.net/$component.git + fi done -for component in $REPOS; do +for component in $REPOS; +do echo "Checking out $component to $BRANCH" git -C $HOME/$component checkout $BRANCH done -if [[ ! -d ./master ]]; then +if [[ ! -d ./master ]]; +then buildbot create-master master cd master ln -s ../deployment/buildbot/master.cfg cd .. fi -if [[ ! -d worker ]]; then +if [[ ! -d worker ]]; +then buildbot-worker create-worker --umask=0o22 ~/worker 127.0.0.1:9988 buildmaster-worker buildmaster-pass fi diff --git a/buildbot/make-buildbot-sitesworker.sh b/buildbot/make-buildbot-sitesworker.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Bootstrap the Taler setup for the sites-worker. + +set -eu + +if ! test "sites-worker" = $(whoami); then + echo Only run as the 'sites-worker' user. + exit 1 +fi + +export PATH=$HOME/.local/bin:$PATH + +pip install --user --break-system-packages buildbot-worker + +if [[ ! -d worker ]]; then + buildbot-worker create-worker --umask=0o22 ~/worker 127.0.0.1:9988 sites-worker sites-pass +fi + +BRANCH=master +REPOS="www twister buywith" + +cd $HOME + +for component in $REPOS; +do + if ! test -d "$HOME/$component" ; + then + git clone "git://git.taler.net/$component.git" + fi +done + +for component in $REPOS; +do + echo "Checking out $component to $BRANCH" + git -C "$HOME/$component" checkout "$BRANCH" +done +mkdir -p ./stamps + +ln -sf ../deployment/worker-sites/Makefile stamps/ +ln -sf ../deployment/worker-sites/invalidate.sh stamps/ + +if [[ ! -d worker ]]; +then + buildbot-worker create-worker --umask=0o22 ~/worker localhost:9988 sites-worker sites-pass +fi diff --git a/buildbot/master.cfg b/buildbot/master.cfg @@ -360,7 +360,7 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( ## # linkchecker worker checks for dead links in the Website -# Location: /home/linkchecker @ taler.net +# Location: /home/linkchecker-worker WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) # linkchecker FACTORY @@ -370,8 +370,8 @@ LINKCHECKER_FACTORY.addStep( name="linkchecker", description="Check taler.net website for broken links && Notify", descriptionDone="Results of wget in buildbot logs.", - command=["/home/linkchecker/deployment/buildbot/linkchecker.sh"], - workdir="/home/linkchecker", + command=["/home/linkchecker-worker/deployment/worker-linkchecker/linkchecker.sh"], + workdir="/home/linkchecker-worker", haltOnFailure=True, timeout=7200 # 2 hours ) diff --git a/codecoverage/with-postgres.sh b/buildbot/with-postgres.sh diff --git a/codespell/dictionary.txt b/codespell/dictionary.txt @@ -1,47 +0,0 @@ -# List of "words" that codespell should ignore in our sources. -# -# Note: The word sensitivity depends on how the to-be-ignored word is -# spelled in codespell_lib/data/dictionary.txt. F.e. if there is a word -# 'foo' and you add 'Foo' _here_, codespell will continue to complain -# about 'Foo'. -# -Nam -BRE -ND -Nd -TE -TEH -UPDATEing -WAN -aci -acn -ba -bre -cant -clen -complet -doas -ect -ehr -fo -hel -ifset -ist -keypair -nd -onl -openin -ot -ser -sie -som -sover -te -te -teh -tha -ths -updateing -vie -wan -wih diff --git a/depo/.config/systemd/user/multi-user.target.wants/bitcoind.service b/depo/.config/systemd/user/multi-user.target.wants/bitcoind.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/bitcoind.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/btc-wire.service b/depo/.config/systemd/user/multi-user.target.wants/btc-wire.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/btc-wire.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/nginx.service b/depo/.config/systemd/user/multi-user.target.wants/nginx.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/nginx.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/taler-local-blog.service b/depo/.config/systemd/user/multi-user.target.wants/taler-local-blog.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/taler-local-blog.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/taler-local-donations.service b/depo/.config/systemd/user/multi-user.target.wants/taler-local-donations.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/taler-local-donations.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/taler-local-exchange.target b/depo/.config/systemd/user/multi-user.target.wants/taler-local-exchange.target @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/taler-local-exchange.target -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/taler-local-landing.service b/depo/.config/systemd/user/multi-user.target.wants/taler-local-landing.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/taler-local-landing.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/taler-local-merchant-backend.service b/depo/.config/systemd/user/multi-user.target.wants/taler-local-merchant-backend.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/taler-local-merchant-backend.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/multi-user.target.wants/wire-gateway.service b/depo/.config/systemd/user/multi-user.target.wants/wire-gateway.service @@ -1 +0,0 @@ -/home/depolymerization/.config/systemd/user/wire-gateway.service -\ No newline at end of file diff --git a/depo/.config/systemd/user/bitcoind.env b/historic/depo/.config/systemd/user/bitcoind.env diff --git a/depo/.config/systemd/user/bitcoind.service b/historic/depo/.config/systemd/user/bitcoind.service diff --git a/depo/.config/systemd/user/btc-wire.service b/historic/depo/.config/systemd/user/btc-wire.service diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/bitcoind.service b/historic/depo/.config/systemd/user/multi-user.target.wants/bitcoind.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/bitcoind.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/btc-wire.service b/historic/depo/.config/systemd/user/multi-user.target.wants/btc-wire.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/btc-wire.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/nginx.service b/historic/depo/.config/systemd/user/multi-user.target.wants/nginx.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/nginx.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-blog.service b/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-blog.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/taler-local-blog.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-donations.service b/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-donations.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/taler-local-donations.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-exchange.target b/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-exchange.target @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/taler-local-exchange.target +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-landing.service b/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-landing.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/taler-local-landing.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-merchant-backend.service b/historic/depo/.config/systemd/user/multi-user.target.wants/taler-local-merchant-backend.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/taler-local-merchant-backend.service +\ No newline at end of file diff --git a/historic/depo/.config/systemd/user/multi-user.target.wants/wire-gateway.service b/historic/depo/.config/systemd/user/multi-user.target.wants/wire-gateway.service @@ -0,0 +1 @@ +/home/depolymerization/.config/systemd/user/wire-gateway.service +\ No newline at end of file diff --git a/depo/.config/systemd/user/nginx.service b/historic/depo/.config/systemd/user/nginx.service diff --git a/depo/.config/systemd/user/taler-local-blog.service b/historic/depo/.config/systemd/user/taler-local-blog.service diff --git a/depo/.config/systemd/user/taler-local-donations.service b/historic/depo/.config/systemd/user/taler-local-donations.service diff --git a/depo/.config/systemd/user/taler-local-exchange-aggregator.service b/historic/depo/.config/systemd/user/taler-local-exchange-aggregator.service diff --git a/depo/.config/systemd/user/taler-local-exchange-closer.service b/historic/depo/.config/systemd/user/taler-local-exchange-closer.service diff --git a/depo/.config/systemd/user/taler-local-exchange-httpd.service b/historic/depo/.config/systemd/user/taler-local-exchange-httpd.service diff --git a/depo/.config/systemd/user/taler-local-exchange-httpd.socket b/historic/depo/.config/systemd/user/taler-local-exchange-httpd.socket diff --git a/depo/.config/systemd/user/taler-local-exchange-secmod-cs.service b/historic/depo/.config/systemd/user/taler-local-exchange-secmod-cs.service diff --git a/depo/.config/systemd/user/taler-local-exchange-secmod-eddsa.service b/historic/depo/.config/systemd/user/taler-local-exchange-secmod-eddsa.service diff --git a/depo/.config/systemd/user/taler-local-exchange-secmod-rsa.service b/historic/depo/.config/systemd/user/taler-local-exchange-secmod-rsa.service diff --git a/depo/.config/systemd/user/taler-local-exchange-transfer.service b/historic/depo/.config/systemd/user/taler-local-exchange-transfer.service diff --git a/depo/.config/systemd/user/taler-local-exchange-wirewatch.service b/historic/depo/.config/systemd/user/taler-local-exchange-wirewatch.service diff --git a/depo/.config/systemd/user/taler-local-exchange.target b/historic/depo/.config/systemd/user/taler-local-exchange.target diff --git a/depo/.config/systemd/user/taler-local-frontends.env b/historic/depo/.config/systemd/user/taler-local-frontends.env diff --git a/depo/.config/systemd/user/taler-local-landing.service b/historic/depo/.config/systemd/user/taler-local-landing.service diff --git a/depo/.config/systemd/user/taler-local-merchant-backend-token.service b/historic/depo/.config/systemd/user/taler-local-merchant-backend-token.service diff --git a/depo/.config/systemd/user/taler-local-merchant-backend.service b/historic/depo/.config/systemd/user/taler-local-merchant-backend.service diff --git a/depo/.config/systemd/user/taler-local-port-redirect.service b/historic/depo/.config/systemd/user/taler-local-port-redirect.service diff --git a/depo/.config/systemd/user/wire-gateway.service b/historic/depo/.config/systemd/user/wire-gateway.service diff --git a/depo/.taler/config/nginx.conf b/historic/depo/.taler/config/nginx.conf diff --git a/depo/.taler/config/taler.conf b/historic/depo/.taler/config/taler.conf diff --git a/depo/.taler/data/.exists b/historic/depo/.taler/data/.exists diff --git a/depo/.taler/sockets/.exists b/historic/depo/.taler/sockets/.exists diff --git a/vagrant/.gitignore b/historic/vagrant/.gitignore diff --git a/vagrant/README.md b/historic/vagrant/README.md diff --git a/vagrant/ubuntu-focal-fossa-build/Vagrantfile b/historic/vagrant/ubuntu-focal-fossa-build/Vagrantfile diff --git a/vagrant/ubuntu/Vagrantfile b/historic/vagrant/ubuntu/Vagrantfile diff --git a/netjail/README b/netjail/README @@ -1 +1,5 @@ -The scripts in this folder are used to run a program in a PID and network namespace. +The scripts in this folder are used to run a program in a PID and network +namespace. + +The buildslave-netjail file must be placed in /etc/sudoers.d/ +to grant the respective users access to the netjail script. +\ No newline at end of file diff --git a/netjail/buildslave-netjail b/netjail/buildslave-netjail @@ -0,0 +1,13 @@ +# For /etc/sudoers.d/ + +# List of users that run a buildslave and thus +# should have the ability to create their own +# network namespace. +# 'grothoff' and 'dold' are listed here to enable them to test things. +# +User_Alias BUILDSLAVE = container-worker, sites-worker, linkchecker-worker + +# The command that we enable running as 'root'. +Cmnd_Alias NETJAIL = /usr/local/bin/netjail.sh + +BUILDSLAVE ALL = NOPASSWD:NETJAIL diff --git a/nlnet/task1/Dockerfile b/nlnet/task1/Dockerfile @@ -1,32 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y openjdk-17-jre git python3-pip curl jq sqlite3 -RUN pip3 install click requests - -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout 9c7079e5323eed4d16e24c1c4245d6586cecac53 # amounts zero-check fixed. -RUN ./bootstrap -RUN ./configure --prefix=/usr/local -RUN make install - -# Reverse proxy -RUN apt-get install -y nginx - -# Importing the UI. -RUN git clone -b prebuilt git://git.taler.net/wallet-core -RUN git -C wallet-core checkout 75af013b348b08b8fb9e65cc9270f2fde964979b # checkout rates fixed. -RUN cp /libeufin/debian/etc/nginx/sites-available/libeufin-sandbox /etc/nginx/sites-enabled/ -RUN mkdir -p /usr/share/libeufin/demobank-ui/ -RUN mkdir -p /etc/libeufin/ -RUN cp /libeufin/debian/usr/share/libeufin/demobank-ui/demobank-ui-settings.js /etc/libeufin/ -RUN cp wallet-core/demobank/* /usr/share/libeufin/demobank-ui/ - -# Default place for the database. -RUN mkdir /libeufin-data - -COPY start.sh / -# ENTRYPOINT /start.sh -CMD /start.sh diff --git a/nlnet/task1/start.sh b/nlnet/task1/start.sh @@ -1,15 +0,0 @@ -#!/bin/bash - -set -eu - -export LIBEUFIN_SANDBOX_ADMIN_PASSWORD=${LIBEUFIN_SANDBOX_ADMIN_PASSWORD:-admin} -export LIBEUFIN_SANDBOX_DB_CONNECTION="jdbc:sqlite:/libeufin-data/libeufin.sqlite" -libeufin-sandbox config --without-registrations --currency ${CURRENCY:-EUR} default -if test -z $LIBEUFIN_EXPOSED_PORT; then - echo ERROR: LIBEUFIN_EXPOSED_PORT is an empty string. - exit 1 -fi - -sed -i "s/localhost/localhost:$LIBEUFIN_EXPOSED_PORT/" /etc/libeufin/demobank-ui-settings.js -service nginx start -libeufin-sandbox serve --port 5016 --no-localhost-only diff --git a/nlnet/task2/Dockerfile b/nlnet/task2/Dockerfile @@ -1,29 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y git - - # python3-pip -# Libeufin Dependencies -RUN apt-get install -y openjdk-17-jre -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout a52cf289234683c4ff492cd8b508cfb6c85ca1e8 -RUN ./bootstrap -RUN apt-get install -y python3-venv -RUN apt-get install -y make -RUN ./configure --prefix=/usr/local -RUN make install -# FIXME: move to the deps block. -RUN apt-get install -y postgresql sudo -RUN grep -v ^host.*all /etc/postgresql/13/main/pg_hba.conf > /tmp/pg_hba_buf.txt -RUN echo "host libeufincheck all 127.0.0.1/32 trust" >> /tmp/pg_hba_buf.txt -RUN echo "host libeufincheck all ::1/128 trust" >> /tmp/pg_hba_buf.txt -RUN cp /tmp/pg_hba_buf.txt /etc/postgresql/13/main/pg_hba.conf -# CMD bash -RUN apt-get install -y jq curl -CMD service postgresql start && \ - sudo -u postgres createuser -s root && \ - createdb -h /var/run/postgresql libeufincheck && \ - make check diff --git a/nlnet/task3/Dockerfile b/nlnet/task3/Dockerfile @@ -1,15 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y openjdk-17-jre git python3-pip curl jq sqlite3 postgresql python3-requests python3-click sudo - -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout 4bc5f38f571a45d427f73813ec3846bf59413afa -RUN ./bootstrap -RUN ./configure --prefix=/usr/local -RUN make install -COPY keys.sh / -COPY start.sh / -ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task3/keys.sh b/nlnet/task3/keys.sh @@ -1,93 +0,0 @@ -#!/bin/bash - -set -eu - -# This script prepares the EBICS keys for one subscriber -# at the PostFinance test platform. - -export LIBEUFIN_NEXUS_DB_CONNECTION="jdbc:sqlite:/tmp/postfinance-nexusdb.sqlite3" - -NEXUS_USERNAME="netzbon-bridge" -NEXUS_PASSWORD="secret" -NEXUS_PORT="5001" - -function exit_cleanup() -{ - for n in `jobs -p` - do - kill $n 2> /dev/null || true - done - wait || true -} - -trap "exit_cleanup" EXIT - -echo "Creating the $NEXUS_USERNAME Nexus user..." -libeufin-nexus superuser $NEXUS_USERNAME --password $NEXUS_PASSWORD -echo "Launching Nexus on port $NEXUS_PORT..." -libeufin-nexus \ - serve --ipv4-only \ - --log-level debug \ - --no-localhost-only \ - --port $NEXUS_PORT > nexus-postfinance.log 2>&1 & - -echo -n "Checking Nexus is serving..." -for i in `seq 1 10`; do - echo -n . - if test i = 10; then - echo Nexus is unreachable - exit 1 - fi - if `curl "http://localhost:$NEXUS_PORT/" &> /dev/null`; then - break - fi - sleep 1 -done -echo OK - -export LIBEUFIN_NEXUS_URL="http://localhost:5001/" -export LIBEUFIN_NEXUS_USERNAME=$NEXUS_USERNAME -export LIBEUFIN_NEXUS_PASSWORD=$NEXUS_PASSWORD - -# FIXME: make connection creation idempotent. -echo "Creating a EBICS connection at Nexus..." -libeufin-cli connections new-ebics-connection \ - --ebics-url https://isotest.postfinance.ch/ebicsweb/ebicsweb \ - --host-id PFEBICS \ - --partner-id $EBICS_PARTNER_ID \ - --ebics-user-id $EBICS_USER_ID \ - --dialect pf \ - postfinanceconn || true - -# 1, send the keys (INI, HIA) -# NOTE: these keys will ONLY be considered if the user -# is in a NEW state, any previous uploaded keys should be reset. -echo "If that is the case, reset any previous keys via the bank Web UI. Press Enter to continue.. " -read -s -echo -n "Sending the new keys to the bank..." -libeufin-cli connections connect postfinanceconn -echo DONE - -# 2, invite the user to unblock them in the Web UI -echo "Please enable the new client keys via the bank Web UI, then press Enter.. " -read -s - -# 3, download the bank keys (HPB). -# That's achieved with another 'connect' action (#7880). -echo -n "Downloading the bank keys..." -libeufin-cli connections connect postfinanceconn -echo DONE -echo "Found the following bank keys:" -libeufin-cli connections show-connection postfinanceconn | jq -r '.details | "Auth: \(.bankAuthKeyHash)\nEnc: \(.bankEncKeyHash)"' - -echo -echo "If any bank keys showed up, please check in the bank Web UI if they match." -echo "If they match, press Enter to continue, otherwise CTRL-C to end." -read -s - -echo -n "Preparing the local keys bundle.." -libeufin-cli connections export-backup \ - --passphrase secret \ - --output-file /tmp/pofi.json \ - postfinanceconn > /dev/null -echo DONE diff --git a/nlnet/task3/salted-incoming-payment-template.csv b/nlnet/task3/salted-incoming-payment-template.csv @@ -1,2 +0,0 @@ -Product;Channel;Account;Currency;Amount;Reference;Name;Street;Number;Postcode;City;Country;DebtorAddressLine;DebtorAddressLine;DebtorAccount;ReferenceType;UltimateDebtorName;UltimateDebtorStreet;UltimateDebtorNumber;UltimateDebtorPostcode;UltimateDebtorTownName;UltimateDebtorCountry;UltimateDebtorAddressLine;UltimateDebtorAddressLine;RemittanceInformationText - QRR;PO;__PAYEE_IBAN__;CHF;33;;D009;Musterstrasse;1;1111;Musterstadt;CH;;;;NON;D009;Musterstrasse;1;1111;Musterstadt;CH;;;__PAYMENT_SALT__ diff --git a/nlnet/task3/start.sh b/nlnet/task3/start.sh @@ -1,79 +0,0 @@ -#!/bin/bash - -set -eu - -function finish() { - exit 1 -} - -trap finish SIGINT - -# Expected arguments are: -# -# $1 := EBICS user ID -# $2 := EBICS partner ID (a.k.a. customer ID) -# $3 := IBAN as assigned by the PostFinance test platform. - -# Suggested invocation via 'docker': -# -# docker run -it $IMAGE_TAG $EBICS_USER_ID $EBICS_PARTNER_ID -service postgresql start -sudo -u postgres createuser -s root -createdb libeufincheck -# This script conducts the key exchange with the bank -# and guides the user to download and upload docuemts -# to the bank. It pauses the execution to let the user -# check and set the Web UI as a double-check mean. - -# Setting the EBICS keys. It'll place them in the container's -# /tmp/pofi.json, where Kotlin expects them. -export EBICS_USER_ID=$1 -export EBICS_PARTNER_ID=$2 -/keys.sh - -# If the keys are ready, it proceeds to invoke the uploading -# and downloading logic. - -# Upload test. - -# The test runner will upload one pain.001 document to -# the bank. Thereafter, the user can check the existtence -# of such document via the bank Web UI. Moreover, the user -# is offered the possibility to specify a custom payment -# subject. - -MY_IBAN=$3 -PAIN_SALT=$RANDOM -echo -echo "Now preparing the pain.001 to upload to the bank via LibEuFin." -echo "This document instructs the bank to send money to an arbitrary" -echo "IBAN by debiting the test platform bank account." -echo "The outgoing payment defaults to have this subject: $PAIN_SALT". -echo "Please enter any value in this prompt, in case you want to" -echo -n "change the default subject: " -read MAYBE_PAIN_SALT - -if ! test "x" = "x$MAYBE_PAIN_SALT"; then - PAIN_SALT=$MAYBE_PAIN_SALT -fi - -echo "The pain.001 will have this subject: '$PAIN_SALT', now calling" -echo "LibEuFin to upload it via EBICS.." -cd /libeufin; ./gradlew -q :nexus:pofi --args="--my-iban \"$MY_IBAN\" upload --subject \"$PAIN_SALT\""; cd - -echo DONE - -echo -echo "Please check the bank Web UI to find the pain.001 document" -echo "whose subject is '$PAIN_SALT'. If that is found, then LibEuFin" -echo "has successfully uploaded it. In the next step, LibEuFin" -echo "will download the new banking records. If '$PAIN_SALT' is found" -echo "in the logs, then it succeeded. Press Enter to continue.. " -read -s - -# Download test. - -# The test runnner proceeds with downloading the banking -# records that witness the payment that was uploaded shortly -# ago. If the logs show the payment subject that belongs -# to such payment, then the download went through. -cd /libeufin; ./gradlew -q :nexus:pofi --args="--my-iban \"$MY_IBAN\" download"; cd - diff --git a/nlnet/task4/Dockerfile b/nlnet/task4/Dockerfile @@ -1,42 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y \ - git \ - openjdk-17-jre \ - python3-pip \ - curl \ - jq \ - postgresql \ - python3-requests \ - python3-click \ - sudo \ - time \ - autoconf \ - autopoint \ - libtool \ - texinfo \ - libgcrypt-dev \ - libidn11-dev \ - zlib1g-dev \ - libunistring-dev \ - libjansson-dev \ - recutils \ - libsqlite3-dev \ - libpq-dev \ - libcurl4-openssl-dev \ - libsodium-dev \ - libqrencode-dev \ - zip - -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout 736c3998648ad249577f8930b616e1f27647f938 -RUN ./bootstrap -RUN ./configure --prefix=/usr/local -RUN make install -RUN make install-nexus -WORKDIR / -COPY start.sh / -ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task4/launch.sh b/nlnet/task4/launch.sh @@ -1,7 +0,0 @@ -#!/bin/bash - -# Binds the container internal port 8080 to the host's. - -set -eu - -docker run -p 8080:8080 -it monitor diff --git a/nlnet/task4/start.sh b/nlnet/task4/start.sh @@ -1,27 +0,0 @@ -#!/bin/bash - -set -eu - -service postgresql start -sudo -u postgres createuser -s root -createdb libeufinbank -cat << EOF > /usr/bin/taler-config -#!/bin/bash - -echo postgresql:///libeufinbank -EOF -chmod +x /usr/bin/taler-config -sed -i 's/ALLOW_CONVERSION = no/ALLOW_CONVERSION = yes/' \ - /libeufin/contrib/libeufin-bank.conf -cat << EOF >> /libeufin/contrib/libeufin-bank.conf - -[nexus-ebics] -currency = EUR -[nexus-postgres] -config = postgresql:///libeufinbank -EOF -libeufin-bank dbinit -c /libeufin/contrib/libeufin-bank.conf -libeufin-nexus dbinit -c /libeufin/contrib/libeufin-bank.conf -/libeufin/contrib/populate-stats.sh /libeufin/contrib/libeufin-bank.conf --one -libeufin-bank passwd admin nlnet -libeufin-bank serve -c /libeufin/contrib/libeufin-bank.conf diff --git a/nlnet/task5/date-range/Dockerfile b/nlnet/task5/date-range/Dockerfile @@ -1,15 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y openjdk-17-jre git python3-pip curl jq sqlite3 postgresql python3-requests python3-click sudo - -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout a614d433a8307468f1074114086ae0a47b848472 -RUN ./bootstrap -RUN ./configure --prefix=/usr/local -RUN make install -COPY start-libeufin.sh / -COPY start.sh / -ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task5/date-range/start-libeufin.sh b/nlnet/task5/date-range/start-libeufin.sh @@ -1,35 +0,0 @@ -DB_CONN="postgresql:///libeufincheck" -export LIBEUFIN_SANDBOX_DB_CONNECTION=$DB_CONN -export LIBEUFIN_NEXUS_DB_CONNECTION=$DB_CONN - -echo -n Delete previous data... -libeufin-sandbox reset-tables -libeufin-nexus reset-tables -echo DONE -echo -n Configure the default demobank with MANA... -libeufin-sandbox config --with-signup-bonus --currency MANA default -echo DONE -echo -n Setting the default exchange at Sandbox... -libeufin-sandbox \ - default-exchange \ - "https://exchange.example.com/" \ - "payto://iban/NOTUSED" -echo DONE -echo -n Start the bank... -export LIBEUFIN_SANDBOX_ADMIN_PASSWORD=foo -libeufin-sandbox serve > sandbox.log 2>&1 & -SANDBOX_PID=$! -echo DONE -echo -n Wait for the bank... -curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5000/ &> /dev/null -echo DONE -echo -n Make one superuser at Nexus... -libeufin-nexus superuser test-user --password x -echo DONE -echo -n Launching Nexus... -libeufin-nexus serve &> nexus.log & -NEXUS_PID=$! -echo DONE -echo -n Waiting for Nexus... -curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5001/ &> /dev/null -echo DONE diff --git a/nlnet/task5/date-range/start.sh b/nlnet/task5/date-range/start.sh @@ -1,155 +0,0 @@ -#!/bin/bash - -# This script shows how Nexus can request histories from -# a particular time frame. Such request must succeed via -# two connection types: EBICS and x-libeufin-bank. EBICS -# ensures the fetching of fiat payments made to the regional -# currency authority, whereas x-libeufin-bank does it for -# the regional currency circuit. Note: the time-framed -# request is exceptional: it's used only after a complaint -# from a user where they didn't get their funds as expected. - -set -eu - -service postgresql start -sudo -u postgres createuser -s root -createdb libeufincheck - -echo -n Launching and confirguring LibEuFin.. -source /start-libeufin.sh &> /dev/null -# Register the Sandbox account. -export LIBEUFIN_SANDBOX_USERNAME=sandbox-user -export LIBEUFIN_SANDBOX_PASSWORD=foo -libeufin-cli \ - sandbox --sandbox-url http://localhost:5000/ \ - demobank \ - register -# x-libeufin-bank connection. -# Creating the x-libeufin-bank connection at Nexus. -export LIBEUFIN_NEXUS_USERNAME=test-user -export LIBEUFIN_NEXUS_PASSWORD=x -export LIBEUFIN_NEXUS_URL=http://localhost:5001 -libeufin-cli connections new-xlibeufinbank-connection \ - --bank-url "http://localhost:5000/demobanks/default/access-api" \ - --username sandbox-user \ - --password foo \ - xlibeufinbankconn -# Connecting the x-libeufin-bank connection... -libeufin-cli connections connect xlibeufinbankconn -# Importing the bank account under a local name at Nexus. -# Importing the x-libeufin-bank account locally.. -libeufin-cli connections import-bank-account \ - --offered-account-id sandbox-user \ - --nexus-bank-account-id foo-at-nexus xlibeufinbankconn - -# EBICS connection. -## Sandbox side. -export LIBEUFIN_SANDBOX_USERNAME=admin -# "Create EBICS host at Sandbox..." -libeufin-cli sandbox \ - --sandbox-url http://localhost:5000 \ - ebicshost create --host-id wwwebics -# Create nlnet EBICS subscriber at Sandbox -libeufin-cli sandbox \ - --sandbox-url http://localhost:5000 \ - demobank new-ebicssubscriber --host-id wwwebics \ - --user-id nlnet --partner-id nlnet \ - --bank-account sandbox-user # that's a username _and_ a bank account name -## Nexus side. -export LIBEUFIN_NEXUS_USERNAME=test-user -export LIBEUFIN_NEXUS_PASSWORD=x -export LIBEUFIN_NEXUS_URL=http://localhost:5001 -# Creating the EBICS connection at Nexus... -libeufin-cli connections new-ebics-connection \ - --ebics-url "http://localhost:5000/ebicsweb" \ - --host-id wwwebics \ - --partner-id nlnet \ - --ebics-user-id nlnet \ - ebicsconn -# Setup EBICS keying... -libeufin-cli connections connect ebicsconn > /dev/null -# Download bank account name from Sandbox... -libeufin-cli connections download-bank-accounts ebicsconn -# Importing bank account info into Nexus... -libeufin-cli connections import-bank-account \ - --offered-account-id sandbox-user \ - --nexus-bank-account-id bar-at-nexus ebicsconn -echo DONE - -FIRST_JAN_2020="1577833200000" # in milliseconds -END_DEC_2019="2019-12-30" -MID_JAN_2020="2020-01-15" - -echo Make sample transaction.. -# 0, setup and start services. -libeufin-sandbox make-transaction \ - --credit-account=admin \ - --debit-account=sandbox-user MANA:2 \ - "task5" # subject. -echo DONE - -echo -n Articifially set the transaction date to $FIRST_JAN_2020.. -# 1, set artificial time for the transaction at January, 1st 2020. -echo "UPDATE bankaccounttransactions SET date='$FIRST_JAN_2020' WHERE subject='task5'" | psql -q -d libeufincheck -echo DONE - -# 2, retrieve the transaction via Nexus, for both -# connections and by asking for a (narrow) time frame -# that includes the 2020-01-01 payment. - -echo -n Nexus: syncing banking records for the time frame $END_DEC_2019-$MID_JAN_2020 via EBICS.. -# Fetch time-framed payments via EBICS. -libeufin-cli \ - accounts \ - fetch-transactions \ - --level=report \ - --range-type=time-range \ - --start=$END_DEC_2019 \ - --end=$MID_JAN_2020 \ - bar-at-nexus > /dev/null # EBICS -echo DONE - -echo Showing the synced data.. -# Now checks if Nexus ingested and shows the -# expected payment. -libeufin-cli \ - accounts \ - transactions \ - bar-at-nexus -echo DONE - -echo Resetting the Nexus database.. -# Bring the database state so that Nexus does not hold any payment. -echo "DELETE FROM nexusbanktransactions" | psql -d libeufincheck -echo "DELETE FROM nexusbankmessages" | psql -d libeufincheck -echo DONE - -echo Checking that no payment data appears after the reset.. -# Double-checking that the future steps start -# without the previous transactions. -libeufin-cli \ - accounts \ - transactions \ - foo-at-nexus # FIXME: put a 204 No Content check? -echo DONE - -# Fetch time-framed payments via x-libeufin-bank. -echo Nexus: syncing banking records for the time frame ${END_DEC_2019}_${MID_JAN_2020} via x-libeufin-bank.. -libeufin-cli \ - accounts \ - fetch-transactions \ - --level=statement \ - --range-type=time-range \ - --start=$END_DEC_2019 \ - --end=$MID_JAN_2020 \ - foo-at-nexus -echo DONE - -# As in the previous case, now Nexus should show -# the 2020-01-01 the payment. -echo Showing the synced data.. -libeufin-cli \ - accounts \ - transactions \ - foo-at-nexus # FIXME: put a 200 OK check? -echo DONE diff --git a/nlnet/task5/long-poll/Dockerfile b/nlnet/task5/long-poll/Dockerfile @@ -1,14 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y openjdk-17-jre git python3-pip curl jq sqlite3 postgresql python3-requests python3-click sudo libgnunet0.19 - -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout 934a73b09b9e9abba348e15ddc058df5bb9cd6a3 -RUN ./bootstrap -RUN ./configure --prefix=/usr/local -RUN make install -COPY start.sh / -ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task5/long-poll/start.sh b/nlnet/task5/long-poll/start.sh @@ -1,134 +0,0 @@ -#!/bin/bash - -set -eu - -service postgresql start -sudo -u postgres createuser -s root -createdb libeufincheck - -wire_transfer () { - RESERVE_PUB=$(gnunet-ecc -g1 /tmp/www &> /dev/null && gnunet-ecc -p /tmp/www) - DB_CONN="postgresql:///libeufincheck" - libeufin-sandbox \ - make-transaction \ - --credit-account=sandbox-user \ - --debit-account=admin MANA:2 \ - $RESERVE_PUB -} - -WITH_TASKS=1 -echo RUNNING SANDBOX-NEXUS EBICS PAIR -jq --version &> /dev/null || (echo "'jq' command not found"; exit 77) -curl --version &> /dev/null || (echo "'curl' command not found"; exit 77) - -DB_CONN="postgresql:///libeufincheck" -export LIBEUFIN_SANDBOX_DB_CONNECTION=$DB_CONN -export LIBEUFIN_NEXUS_DB_CONNECTION=$DB_CONN - -echo -n Delete previous data... -libeufin-sandbox reset-tables -libeufin-nexus reset-tables -echo DONE -echo -n Configure the default demobank with MANA... -libeufin-sandbox config --with-signup-bonus --currency MANA default -echo DONE -echo -n Setting the default exchange at Sandbox... -libeufin-sandbox \ - default-exchange \ - "https://exchange.example.com/" \ - "payto://iban/NOTUSED" -echo DONE -echo -n Start the bank... -export LIBEUFIN_SANDBOX_ADMIN_PASSWORD=foo -libeufin-sandbox serve > sandbox.log 2>&1 & -SANDBOX_PID=$! -echo DONE -echo -n Wait for the bank... -curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5000/ &> /dev/null -echo DONE -echo -n Make one superuser at Nexus... -libeufin-nexus superuser test-user --password x -echo DONE -echo -n Launching Nexus... -libeufin-nexus serve &> nexus.log & -NEXUS_PID=$! -echo DONE -echo -n Waiting for Nexus... -curl --max-time 4 --retry-all-errors --retry-connrefused --retry-delay 1 --retry 10 http://localhost:5001/ &> /dev/null -echo DONE - -echo -n "Register the Sandbox account..." -export LIBEUFIN_SANDBOX_USERNAME=sandbox-user -export LIBEUFIN_SANDBOX_PASSWORD=foo -libeufin-cli \ - sandbox --sandbox-url http://localhost:5000/ \ - demobank \ - register -echo DONE -echo -n Creating the x-libeufin-bank connection at Nexus... -export LIBEUFIN_NEXUS_USERNAME=test-user -export LIBEUFIN_NEXUS_PASSWORD=x -export LIBEUFIN_NEXUS_URL=http://localhost:5001 -# echoing the password to STDIN, as that is a "prompt" option. -libeufin-cli connections new-xlibeufinbank-connection \ - --bank-url "http://localhost:5000/demobanks/default/access-api" \ - --username sandbox-user \ - --password foo \ - wwwconn -echo DONE -echo -n Connecting the x-libeufin-bank connection... -libeufin-cli connections connect wwwconn -echo DONE -# Importing the bank account under a local name at Nexus. -echo -n Importing the x-libeufin-bank account locally.. -libeufin-cli connections import-bank-account \ - --offered-account-id sandbox-user \ - --nexus-bank-account-id foo-at-nexus wwwconn -echo DONE -echo -n Create the Taler facade at Nexus... -libeufin-cli facades \ - new-taler-wire-gateway-facade \ - --currency TESTKUDOS --facade-name test-facade \ - wwwconn foo-at-nexus -echo DONE -if test 1 = $WITH_TASKS; then - echo -n Creating submit transactions task.. - libeufin-cli accounts task-schedule \ - --task-type submit \ - --task-name www-payments \ - --task-cronspec "* * *" \ - foo-at-nexus || true - # Tries every second. Ask C52 - echo DONE - echo -n Creating fetch transactions task.. - # Not idempotent, FIXME #7739 - libeufin-cli accounts task-schedule \ - --task-type fetch \ - --task-name www-history \ - --task-cronspec "* * *" \ - --task-param-level statement \ - --task-param-range-type since-last \ - foo-at-nexus || true - echo DONE -else - echo NOT creating background tasks! -fi - -echo -echo Services are online! The following shell offers a 'wire_transfer' -echo command that wires money to Nexus 'test-user'. Give it after having -echo connected an HTTP client that long-polls to Nexus. As an example, a -echo 100 seconds long-poller to Nexus is the following command: -echo curl -v -u test-user:x "'http://localhost:5001/facades/test-facade/taler-wire-gateway/history/incoming?delta=5&long_poll_ms=100000'" -echo -echo Hint: after having issued the previous command and having observed -echo that it actually long-polls, press CTRL-Z to send it in the background, -echo "then wire the funds to the long-poller with 'wire_transfer'," -echo "and finally give 'fg 1' to bring the long-poller in the foreground." -echo If the client now shows a response, then the long-polling mechanism -echo worked. -echo - -cd / -export -f wire_transfer -bash diff --git a/nlnet/task5/performance/Dockerfile b/nlnet/task5/performance/Dockerfile @@ -1,70 +0,0 @@ -FROM debian:stable - -RUN apt-get update -RUN apt-get install -y \ - git \ - openjdk-17-jre \ - python3-pip \ - curl \ - jq \ - postgresql \ - python3-requests \ - python3-click \ - sudo \ - time \ - autoconf \ - autopoint \ - libtool \ - texinfo \ - libgcrypt-dev \ - libidn11-dev \ - zlib1g-dev \ - libunistring-dev \ - libjansson-dev \ - recutils \ - libsqlite3-dev \ - libpq-dev \ - libcurl4-openssl-dev \ - libsodium-dev \ - libqrencode-dev \ - zip - -# Installation -RUN git clone git://git.taler.net/libeufin -WORKDIR /libeufin -RUN git fetch && git checkout 4bc5f38f571a45d427f73813ec3846bf59413afa -RUN ./bootstrap -RUN ./configure --prefix=/usr/local -RUN make install -WORKDIR / -RUN git clone git://git.gnunet.org/libmicrohttpd -WORKDIR /libmicrohttpd -RUN ./bootstrap -RUN ./configure --disable-doc -RUN make install -WORKDIR / -RUN git clone git://git.gnunet.org/gnunet -WORKDIR /gnunet -RUN apt-get install -y python3-sphinx python3-sphinx-rtd-theme # Move up? -RUN ./bootstrap -RUN ./configure -RUN pip3 install --break-system-packages htmlark -RUN make install -WORKDIR / -RUN git clone git://git.taler.net/exchange -WORKDIR /exchange -RUN ./bootstrap -RUN ./configure -RUN make install -WORKDIR / -RUN git clone git://git.taler.net/merchant -WORKDIR /merchant -RUN ./bootstrap -RUN ./configure -RUN make install -WORKDIR / - -COPY start.sh / -RUN apt-get install -y wget -RUN apt-get install -y bc -ENTRYPOINT ["/start.sh"] diff --git a/nlnet/task5/performance/start.sh b/nlnet/task5/performance/start.sh @@ -1,107 +0,0 @@ -#!/bin/bash - -# This script shows, via runnuing the benchmark, how -# the LibEuFin database connections are significantly -# shorter than the benchmark total time. - -# For this reason, it can only be that LibEuFin opens -# and closes many PostgreSQL connections, as it is required -# by milestone #3. - -set -eu - -export HOW_MANY_WITHDRAWALS=100 - -service postgresql start -sudo -u postgres createuser -s root - -# Activating the disconnection logs. -sudo -u postgres psql -q -c "ALTER SYSTEM SET log_disconnections = 'on'" -c "SELECT pg_reload_conf()" > /dev/null - -# Converts AA:BB:CC.DDD to milliseconds. -convert_pg_time_to_ms () { - awk -F[.:] '{SECS=(60*60*$1)+(60*$2)+$3; MILLI=$4; TOTAL_MS=(SECS*1000)+MILLI; print TOTAL_MS}' -} - -createdb talercheck -export LD_LIBRARY_PATH=/usr/local/lib - -prepare_and_run () { - taler-unified-setup.sh \ - -Wwemtns \ - -c /exchange/src/benchmark/benchmark-cs.conf \ - -u exchange-account-2 &> /check_ready.txt & - # Wait that the prep. went through. - echo -n Waiting the unified setup to complete.. - READY="NO" - for i in `seq 100` true; do - if grep -q "<<READY>>" /check_ready.txt; then - READY="YES" - break - fi - echo -n "."; sleep 1 - done - - if test $READY = "YES"; then - echo "DONE" - else - cat /check_ready.txt - echo FAIL - exit 1 - fi - - echo Running the benchmark.. - taler-exchange-benchmark \ - -c /exchange/src/benchmark/benchmark-cs.conf.edited \ - -u exchange-account-2 \ - -L WARNING \ - -n 1 \ - -r $HOW_MANY_WITHDRAWALS -} - -export -f prepare_and_run -/usr/bin/time -o /benchmark-wall-clock-time.txt --format=%e bash -c "prepare_and_run" - -NEXUS_PID=$(cat /libeufin-nexus.pid) -SANDBOX_PID=$(cat /libeufin-sandbox.pid) - -if test -z $NEXUS_PID; then - echo Could not find Nexus PID, failing. - exit 1 -fi - -if test -z $SANDBOX_PID; then - echo Could not find Sandbox PID, failing. - exit 1 -fi - -# Convert the wall clock time to milliseconds, to make -# it compatible with the format as GREPped through Postgres logs. -BENCHMARK_TOT_MS=$(awk -F. '{t=($1 * 1000 + $2 * 10)} END {print t}' /benchmark-wall-clock-time.txt) - -NEXUS_LONGEST_DB_SESSION_MS=$(grep disconnection < /var/log/postgresql/postgresql-15-main.log | grep $NEXUS_PID | grep -o "session time:.*$" | grep -o [0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9] | convert_pg_time_to_ms | sort -n | tail -n 1) - -SANDBOX_LONGEST_DB_SESSION_MS=$(grep disconnection < /var/log/postgresql/postgresql-15-main.log | grep $SANDBOX_PID | grep -o "session time:.*$" | grep -o [0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9][0-9] | convert_pg_time_to_ms | sort -n | tail -n 1) - -if test $NEXUS_LONGEST_DB_SESSION_MS -gt $BENCHMARK_TOT_MS; then - echo Nexus had a DB session longer than the benchmark itself, failing. - exit 1 -fi - -if test $SANDBOX_LONGEST_DB_SESSION_MS -gt $BENCHMARK_TOT_MS; then - echo Sandbox had a DB session longer than the benchmark itself, failing. - exit 1 -fi - -NEXUS_TIME_PORTION=$(echo "($NEXUS_LONGEST_DB_SESSION_MS / $BENCHMARK_TOT_MS) * 100" | bc -lq | sed 's/^\./0./') -SANDBOX_TIME_PORTION=$(echo "($SANDBOX_LONGEST_DB_SESSION_MS / $BENCHMARK_TOT_MS) * 100" | bc -lq | sed 's/^\./0./') - -# Here: the further from 1 the better. -echo Nexus longest DB session is $NEXUS_TIME_PORTION percent of the total benchmark time. -echo Sandbox longest DB session is $SANDBOX_TIME_PORTION percent of the total benchmark time. - -# Now show the total space occupied by the database. -# Although that's a _total_ estimate, it'll anyhow show -# that _also_ libeufin has reasonable data usage. -TOTAL_DB_SPACE=$(echo "SELECT pg_size_pretty(pg_database_size('talercheck'))" | psql -d talercheck | grep "^ [0-9]" | tr -d "[:blank:]") -echo "The total space occupied by the database for $HOW_MANY_WITHDRAWALS withdrawals is $TOTAL_DB_SPACE" diff --git a/pylint/README b/pylint/README @@ -1,5 +0,0 @@ -rc file to drive pylint; mainly used to disable certain -warning which aren't critical to our applications. - -We added the "missing docstring" warning to the ignored -ones. diff --git a/pylint/pylintrc b/pylint/pylintrc @@ -1,425 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS,migrations,south_migrations,wsgi.py - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,missing-docstring,logging-not-lazy,import-error - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[BASIC] - -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression matching correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|(urls|urlpatterns|register))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_,qs - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )?<?https?://\S+>?$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception diff --git a/gnu/release-template.txt b/sign-gnu-artefact/release-template.txt diff --git a/gnu/sign-gnu-artefact b/sign-gnu-artefact/sign-gnu-artefact diff --git a/gnu/upload-gnu-artefact b/sign-gnu-artefact/upload-gnu-artefact diff --git a/systemd-services/buildbot-worker-linkchecker.service b/systemd-services/buildbot-worker-linkchecker.service @@ -1,12 +1,14 @@ [Unit] -Description=Buildbot worker service for doc -AssertPathExists=/home/linkchecker/worker +Description=Buildbot worker service for checking for dead links +AssertPathExists=/home/linkchecker-worker/worker [Service] -WorkingDirectory=/home/linkchecker/ -ExecStart=/usr/bin/buildbot-worker start --nodaemon worker -ExecReload=/usr/bin/buildbot-worker restart --nodaemon worker -ExecStop=/usr/bin/buildbot-worker stop worker +User=linkchecker-worker +Group=linkchecker-worker +WorkingDirectory=~ +ExecStart=/home/linkchecker-worker/.local/buildbot-worker start --nodaemon worker +ExecReload=/home/linkchecker-worker/.local/buildbot-worker restart --nodaemon worker +ExecStop=/home/linkchecker-worker/.local/buildbot-worker stop worker Restart=always [Install] diff --git a/systemd-services/buildbot-worker-sites.service b/systemd-services/buildbot-worker-sites.service @@ -1,12 +1,14 @@ [Unit] -Description=Buildbot worker service for sites -AssertPathExists=/home/taler-websites/worker +Description=Buildbot worker service for updating the Web sites +AssertPathExists=/home/sites-worker/worker [Service] -WorkingDirectory=/home/taler-websites/ -ExecStart=/usr/bin/buildbot-worker start --nodaemon worker -ExecReload=/usr/bin/buildbot-worker restart --nodaemon worker -ExecStop=/usr/bin/buildbot-worker stop worker +User=sites-worker +Group=sites-worker +WorkingDirectory=~ +ExecStart=/home/sites-worker/.local/bin/buildbot-worker start --nodaemon worker +ExecReload=/home/sites-worker/.local/bin/buildbot-worker restart --nodaemon worker +ExecStop=/home/sites-worker/.local/bin/buildbot-worker stop worker Restart=always [Install] diff --git a/taler-sitesbuild/Makefile b/taler-sitesbuild/Makefile @@ -1,22 +0,0 @@ -BASE="${HOME}/deployment/taler-sitesbuild" - -all: www.taler.net-stamp -all: twister.taler.net-stamp -all: stage.taler.net-stamp -all: buywith.taler.net-stamp - -stage.taler.net-stamp: - ${BASE}/update_stage.sh - touch $@ - -www.taler.net-stamp: - ${BASE}/update_www.sh - touch $@ - -twister.taler.net-stamp: - ${BASE}/update_twister_page.sh - touch $@ - -buywith.taler.net-stamp: - ${BASE}/update_buywith_page.sh - touch $@ diff --git a/codecoverage/coverage.sh b/worker-codecoverage/coverage.sh diff --git a/codecoverage/run-coverage-helper.sh b/worker-codecoverage/run-coverage-helper.sh diff --git a/buildbot/linkchecker.Containerfile b/worker-linkchecker/linkchecker.Containerfile diff --git a/worker-linkchecker/linkchecker.sh b/worker-linkchecker/linkchecker.sh @@ -0,0 +1,44 @@ +#!/bin/bash +#set -v +# Removed because wget errors with error 8 (Server issued an error response.) +#set -e + +## This script will scan www.taler.net for broken links and e-mail a copy of the log if any are found. + +logfile="$HOME/linkchecker.log" +wait_time="1" +recurse_level="1" +ignore_list="(.*)demo.taler.net(.*)\/orders\/(.*)" # appears to do *nothing* + +# Remove old log + +if [ -f "$logfile" ] + then + echo $logfile " exists. Moving to" $logfile".old." + mv $logfile $logfile.old + else + echo "Info: existing log file '$logfile' not found." +fi + +podman build -t linkchecker:latest -f "$HOME/deployment/worker-linkchecker/linkchecker.Containerfile" "$HOME/deployment/worker-linkchecker" + +# Use wget to scan hosts and save output +for url in "https://www.taler.net/" "https://docs.taler.net/" "https://taler-systems.com/" "https://demo.taler.net/" "https://bank.demo.taler.net/" "https://shop.demo.taler.net/" "https://donations.demo.taler.net/" ; do + echo -e "\n\n#############################\n## Starting check on ${url}\n#############################\n" + podman run --rm localhost/linkchecker:latest \ + linkchecker \ + --no-robots \ + --check-extern \ + --recursion-level="$recurse_level" \ + "$url" | tee --append "$logfile" +done + +# Search the log for the phrase "broken link" as this is what wget will report +if grep -Rl 'Error' $logfile + then + echo "Found broken links. Build should fail (exit 1), triggering e-mail notification." + exit 1 + else + echo "No broken links found. Nothing more to do." + exit 0 +fi diff --git a/buildbot/linkcheckerrc b/worker-linkchecker/linkcheckerrc diff --git a/worker-sites/Makefile b/worker-sites/Makefile @@ -0,0 +1,22 @@ +BASE="${HOME}/deployment/worker-sitesbuild" + +all: www.taler.net-stamp +all: twister.taler.net-stamp +all: stage.taler.net-stamp +all: buywith.taler.net-stamp + +stage.taler.net-stamp: + ${BASE}/update_stage.sh + touch $@ + +www.taler.net-stamp: + ${BASE}/update_www.sh + touch $@ + +twister.taler.net-stamp: + ${BASE}/update_twister_page.sh + touch $@ + +buywith.taler.net-stamp: + ${BASE}/update_buywith_page.sh + touch $@ diff --git a/taler-sitesbuild/invalidate.sh b/worker-sites/invalidate.sh diff --git a/taler-sitesbuild/update_buywith_page.sh b/worker-sites/update_buywith_page.sh diff --git a/taler-sitesbuild/update_stage.sh b/worker-sites/update_stage.sh diff --git a/taler-sitesbuild/update_twister_page.sh b/worker-sites/update_twister_page.sh diff --git a/taler-sitesbuild/update_www.sh b/worker-sites/update_www.sh