document deployment better, create systemd service for main test job, unify some systemd scripts via %h

9 files changed, 89 insertions, 56 deletions

diff --git a/README.md b/README.md
index ab8d8fe..0def23d 100644
--- a/README.md
+++ b/README.md
@@ -1,26 +1,27 @@
 Deployment Layout
 =================
 
-
-bin/ -- scripts to manage a taler deployment (test/demo/int)
-buildbot/ -- scripts for the buildmaster and jobs launched by it
+bin/ -- main scripts to manage a taler deployment (test/demo/int)
+buildbot/ -- scripts and configuration for the buildmaster and jobs launched by it
 envcfg/ -- Git tags for various deployments, specify which versions should be used
 history/ -- directory for NOT MAINTAINED scripts
 historic/docker/ -- dockerized Taler deployment
 historic/guix/ -- Guix packages for Taler
-mypy/ -- MARCELLO?
 netjail/ -- scripts to create a namespaces to ensure buildslaves do not compete for resources
 postmortem/ -- postmortem reports about major incidents we have had with lessons learned
 private-keys/ -- deployment private keys so that keys do not change if we migrate elsewhere
-pylint/ -- MARCELLO?
-selenium/ -- MARCELLO?
 sign-gnu-artefact -- shell script for making releases and preparing upload to ftp.gnu.org
 systemd-services/ -- service definitions for buildbot build-slaves
 taler-arm/ -- gnunet-arm configurations for a deployment
-
 taler-sitesbuild/ -- scripts for updating our Web sites
-taler-walletbuild/ -- scripts for building the Taler wallet
 
-TOOD:
-- move scripts like taler-sitesbuild and taler-walletbuild into the
-  respective Git repositories and/or master.cfg
+TODO:
+
+bootstrap-* -- shell scripts to setup the accounts to run some of the services on taler.net; TODO: should likely be integrated with "$ bin/taler-deployment bootstrap" logic for uniformity
+
+
+mypy/ -- MARCELLO?
+prepare-* -- MARCELLO?
+pylint/ -- MARCELLO?
+selenium/ -- MARCELLO?
+build-reservetopper -- MARCELLO?
diff --git a/bin/taler-deployment b/bin/taler-deployment
index 1b76dbf..ff0be67 100755
--- a/bin/taler-deployment
+++ b/bin/taler-deployment
@@ -25,6 +25,7 @@ import time
 from pathlib import Path
 from dataclasses import dataclass
 from typing import List, Callable
+from shutil import copy
 
 activate_template = """\
 #!/bin/bash
@@ -43,6 +44,7 @@ export TALER_ENV_URL_MERCHANT_DONATIONS="https://donations.{envname}.taler.net/"
 export TALER_ENV_URL_MERCHANT_SURVEY="https://survey.{envname}.taler.net/"
 export TALER_ENV_URL_AUDITOR="https://auditor.{envname}.taler.net/"
 export TALER_ENV_URL_BACKOFFICE="https://backoffice.{envname}.taler.net/"
+export TALER_ENV_URL_SYNC="https://sync.{envname}.taler.net/"
 export TALER_COVERAGE={coverage}
 """
 
@@ -296,8 +298,15 @@ def get_repos(envname):
                 ["exchange", "libmicrohttpd"],
                 build_merchant,
             ),
+            Repo(
+                "sync",
+                "git://git.taler.net/sync",
+                ["exchange", "merchant"],
+                build_sync,
+            ),
         ]
 
+    # Note: these are currently not in use!
     if envname in ("euro", "chf"):
         return [
             Repo(
@@ -553,13 +562,13 @@ def bootstrap() -> None:
             (home / "taler-data").symlink_to("/home/demo/shared-data")
 
     if envname == "test":
-        create_bb_worker("bb-worker", "test-worker", "test-pass")
+        create_bb_worker("buildbot-worker-taler.service", "bb-worker", "test-worker", "test-pass")
     elif envname == "auditor-reporter":
-        create_bb_worker("bb-worker", "auditor-worker", "auditor-pass")
+        create_bb_worker("buildbot-worker-auditor.service", "auditor-worker", "auditor-pass")
     elif envname == "demo-checker":
-        create_bb_worker("bb-worker", "demo-worker", "demo-pass")
+        create_bb_worker("buildbot-worker-taler-healthcheck.service", "bb-worker", "demo-worker", "demo-pass")
     elif envname == "coverage":
-        create_bb_worker("bb-worker", "lcov-worker", "lcov-pass")
+        create_bb_worker("buildbot-worker-lcov.service", "worker", "lcov-worker", "lcov-pass")
 
         www_path = Path.home() / "www"
         www_path.mkdir(exist_ok=True)
@@ -573,12 +582,17 @@ def bootstrap() -> None:
                 Path.home() / "sources" / "exchange" / "coverage_report",
                 www_path / "exchange",
             )
+        if not os.path.islink(www_path / "sync"):
+            os.symlink(
+                Path.home() / "sources" / "sync" / "coverage_report",
+                www_path / "sync",
+            )
 
     print("Bootstrap finished.")
     print("Please source the ~/activate file before proceeding.")
 
 
-def create_bb_worker(dirname, workername, workerpw):
+def create_bb_worker(systemd_unit, dirname, workername, workerpw):
     home = Path.home()
     bb_dir = home / dirname
     if bb_dir.exists():
@@ -595,7 +609,40 @@ def create_bb_worker(dirname, workername, workerpw):
         ],
         check=True,
     )
+    setup_service (systemd_unit)
+
 
+def setup_service(systemd_unit):
+    sc_path = Path.home() / ".config" / "systemd" / "user"
+    sc_path.mkdir(exist_ok=True)
+    sc_unit = Path.home() / "deployment" / "systemd-services" / systemd_unit
+    copy(sc_unit, sc_path)
+    subprocess.run(
+        [
+            "systemctl",
+            "--user",
+            "daemon-reload",
+        ],
+        check=True,
+    )
+    subprocess.run(
+        [
+            "systemctl",
+            "--user",
+            "enable",
+            systemd_unit
+        ],
+        check=True,
+    )
+    subprocess.run(
+        [
+            "systemctl",
+            "--user",
+            "start",
+            systemd_unit
+        ],
+        check=True,
+    )
 
 if __name__ == "__main__":
     cli()
diff --git a/buildbot/build.sh b/buildbot/build.sh
index 60e4ed5..5a1e8e9 100755
--- a/buildbot/build.sh
+++ b/buildbot/build.sh
@@ -2,14 +2,6 @@
 
 set -eu
 
-PAUSE_LOCKFILE=/tmp/buildbot_pause.lock
-
-if [ -f $PAUSE_LOCKFILE ]; then
-    echo "The Buildbot is paused, remove ${PAUSE_LOCKFILE} to resume."
-    # Treat this as error, so dependent schedulers won't fire up.
-    exit 1
-fi
-
 echo "Running taler-deployment bootstrap"
 source "${HOME}/activate"
 taler-deployment bootstrap
diff --git a/buildbot/coverage.sh b/buildbot/coverage.sh
index e3a22e3..e8d0db4 100755
--- a/buildbot/coverage.sh
+++ b/buildbot/coverage.sh
@@ -2,25 +2,31 @@
 
 set -eu
 
-PAUSE_LOCKFILE=/tmp/buildbot_pause.lock
+source "${HOME}/activate"
 
-if [ -f $PAUSE_LOCKFILE ]; then
-    echo "The Buildbot is paused, remove ${PAUSE_LOCKFILE} to resume."
-    # Treat this as error, so dependent schedulers won't fire up.
-    exit 1
-fi
+PG_VER=12
+PG_DIR=/usr/lib/postgresql/$PG_VER/bin
 
-source "${HOME}/activate"
+# setup little postgres DB for us ...
+TMP_DB_DIR=`mktemp tmpXXXXXX`
+$PG_DIR/initdb -D $TMP_DB_DIR/ -A trust
+$PG_DIR/createdb talercheck
+$PG_DIR/createdb synccheck
+$PG_DIR/createdb anastasischeck
+$PG_DIR/pg_ctl -D $TMP_DB_DIR/ -l logfile start
 
-for codebase in merchant exchange ; do
+for codebase in merchant exchange sync ; do
   TOP="$HOME/sources/${codebase}/"
   REPORT_DIR="$HOME/sources/${codebase}/coverage_report"
   lcov -d $TOP -z
   cd $TOP
-  make check || true
+  make check -k || true
   mkdir -p $REPORT_DIR
   lcov -d $TOP -c --no-external -o $REPORT_DIR/coverage.info
   lcov -r $REPORT_DIR/coverage.info **/test_* -o $REPORT_DIR/rcoverage.info
   genhtml -o $REPORT_DIR $REPORT_DIR/rcoverage.info
   chmod a+rx -R $REPORT_DIR
 done
+
+$PG_DIR/pg_ctl -D $TMP_DB_DIR/ stop
+rm -rf $TMP_DB_DIR
diff --git a/envcfg.py.template b/envcfg.py.template
index c7523ca..8957a79 100644
--- a/envcfg.py.template
+++ b/envcfg.py.template
@@ -1,4 +1,4 @@
-# Name of the environment (test, demo, int, ...)
+# Name of the environment (test, demo, int, coverage, demo-checker, auditor-reporter, ...)
 env = "..."
 
 tag = "master"
diff --git a/systemd-services/buildbot-worker-demo-auditor.service b/systemd-services/buildbot-worker-auditor.service
index 1151b89..8e350e1 100644
--- a/systemd-services/buildbot-worker-demo-auditor.service
+++ b/systemd-services/buildbot-worker-auditor.service
@@ -1,9 +1,9 @@
 [Unit]
-Description=Buildbot worker service for demo-auditor
-AssertPathExists=/home/demo-auditor/worker
+Description=Buildbot worker service for auditor
+AssertPathExists=%h/worker
 
 [Service]
-WorkingDirectory=/home/demo-auditor/
+WorkingDirectory=%h
 ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
 ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
 ExecStop=/usr/local/bin/buildbot-worker stop worker
diff --git a/systemd-services/buildbot-worker-test-auditor.service b/systemd-services/buildbot-worker-taler-healthcheck.service
index 6cb1429..00ab298 100644
--- a/systemd-services/buildbot-worker-test-auditor.service
+++ b/systemd-services/buildbot-worker-taler-healthcheck.service
@@ -1,9 +1,9 @@
 [Unit]
-Description=Buildbot worker service for test-auditor
-AssertPathExists=/home/test-auditor/worker
+Description=Buildbot worker service for taler-healthcheck
+AssertPathExists=%h/worker
 
 [Service]
-WorkingDirectory=/home/test-auditor/
+WorkingDirectory=%h
 ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
 ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
 ExecStop=/usr/local/bin/buildbot-worker stop worker
diff --git a/systemd-services/buildbot-worker-taler-test-healthcheck.service b/systemd-services/buildbot-worker-taler-test-healthcheck.service
deleted file mode 100644
index 1973640..0000000
--- a/systemd-services/buildbot-worker-taler-test-healthcheck.service
+++ /dev/null
@@ -1,13 +0,0 @@
-[Unit]
-Description=Buildbot worker service for taler-test-healthcheck
-AssertPathExists=/home/taler-test-healthcheck/worker
-
-[Service]
-WorkingDirectory=/home/taler-test-healthcheck/
-ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
-ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
-ExecStop=/usr/local/bin/buildbot-worker stop worker
-Restart=always
-
-[Install]
-WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-taler-demo-healthcheck.service b/systemd-services/buildbot-worker-taler.service
index ce9ab42..460298b 100644
--- a/systemd-services/buildbot-worker-taler-demo-healthcheck.service
+++ b/systemd-services/buildbot-worker-taler.service
@@ -1,9 +1,9 @@
 [Unit]
-Description=Buildbot worker service for taler-demo-healthcheck
-AssertPathExists=/home/taler-demo-healthcheck/worker
+Description=Buildbot worker service for the main deployment (runs exchange, merchant, sync)
+AssertPathExists=%h/bb-worker
 
 [Service]
-WorkingDirectory=/home/taler-demo-healthcheck/
+WorkingDirectory=%h
 ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
 ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
 ExecStop=/usr/local/bin/buildbot-worker stop worker