commit 7dacf67f5bde548220265fd70650be425c4e4ee0
parent baf34c9e9145f53cb06d2ee63bec3c43af687f02
Author: Christian Grothoff <christian@grothoff.org>
Date: Sun, 23 Feb 2020 13:32:49 +0100
document deployment better, create systemd service for main test job, unify some systemd scripts via %h
Diffstat:
12 files changed, 119 insertions(+), 86 deletions(-)
diff --git a/README.md b/README.md
@@ -1,26 +1,27 @@
Deployment Layout
=================
-
-bin/ -- scripts to manage a taler deployment (test/demo/int)
-buildbot/ -- scripts for the buildmaster and jobs launched by it
+bin/ -- main scripts to manage a taler deployment (test/demo/int)
+buildbot/ -- scripts and configuration for the buildmaster and jobs launched by it
envcfg/ -- Git tags for various deployments, specify which versions should be used
history/ -- directory for NOT MAINTAINED scripts
historic/docker/ -- dockerized Taler deployment
historic/guix/ -- Guix packages for Taler
-mypy/ -- MARCELLO?
netjail/ -- scripts to create a namespaces to ensure buildslaves do not compete for resources
postmortem/ -- postmortem reports about major incidents we have had with lessons learned
private-keys/ -- deployment private keys so that keys do not change if we migrate elsewhere
-pylint/ -- MARCELLO?
-selenium/ -- MARCELLO?
sign-gnu-artefact -- shell script for making releases and preparing upload to ftp.gnu.org
systemd-services/ -- service definitions for buildbot build-slaves
taler-arm/ -- gnunet-arm configurations for a deployment
-
taler-sitesbuild/ -- scripts for updating our Web sites
-taler-walletbuild/ -- scripts for building the Taler wallet
-TOOD:
-- move scripts like taler-sitesbuild and taler-walletbuild into the
- respective Git repositories and/or master.cfg
+TODO:
+
+bootstrap-* -- shell scripts to setup the accounts to run some of the services on taler.net; TODO: should likely be integrated with "$ bin/taler-deployment bootstrap" logic for uniformity
+
+
+mypy/ -- MARCELLO?
+prepare-* -- MARCELLO?
+pylint/ -- MARCELLO?
+selenium/ -- MARCELLO?
+build-reservetopper -- MARCELLO?
diff --git a/bin/taler-deployment b/bin/taler-deployment
@@ -25,6 +25,7 @@ import time
from pathlib import Path
from dataclasses import dataclass
from typing import List, Callable
+from shutil import copy
activate_template = """\
#!/bin/bash
@@ -43,6 +44,7 @@ export TALER_ENV_URL_MERCHANT_DONATIONS="https://donations.{envname}.taler.net/"
export TALER_ENV_URL_MERCHANT_SURVEY="https://survey.{envname}.taler.net/"
export TALER_ENV_URL_AUDITOR="https://auditor.{envname}.taler.net/"
export TALER_ENV_URL_BACKOFFICE="https://backoffice.{envname}.taler.net/"
+export TALER_ENV_URL_SYNC="https://sync.{envname}.taler.net/"
export TALER_COVERAGE={coverage}
"""
@@ -296,8 +298,15 @@ def get_repos(envname):
["exchange", "libmicrohttpd"],
build_merchant,
),
+ Repo(
+ "sync",
+ "git://git.taler.net/sync",
+ ["exchange", "merchant"],
+ build_sync,
+ ),
]
+ # Note: these are currently not in use!
if envname in ("euro", "chf"):
return [
Repo(
@@ -553,13 +562,13 @@ def bootstrap() -> None:
(home / "taler-data").symlink_to("/home/demo/shared-data")
if envname == "test":
- create_bb_worker("bb-worker", "test-worker", "test-pass")
+ create_bb_worker("buildbot-worker-taler.service", "bb-worker", "test-worker", "test-pass")
elif envname == "auditor-reporter":
- create_bb_worker("bb-worker", "auditor-worker", "auditor-pass")
+ create_bb_worker("buildbot-worker-auditor.service", "auditor-worker", "auditor-pass")
elif envname == "demo-checker":
- create_bb_worker("bb-worker", "demo-worker", "demo-pass")
+ create_bb_worker("buildbot-worker-taler-healthcheck.service", "bb-worker", "demo-worker", "demo-pass")
elif envname == "coverage":
- create_bb_worker("bb-worker", "lcov-worker", "lcov-pass")
+ create_bb_worker("buildbot-worker-lcov.service", "worker", "lcov-worker", "lcov-pass")
www_path = Path.home() / "www"
www_path.mkdir(exist_ok=True)
@@ -573,12 +582,17 @@ def bootstrap() -> None:
Path.home() / "sources" / "exchange" / "coverage_report",
www_path / "exchange",
)
+ if not os.path.islink(www_path / "sync"):
+ os.symlink(
+ Path.home() / "sources" / "sync" / "coverage_report",
+ www_path / "sync",
+ )
print("Bootstrap finished.")
print("Please source the ~/activate file before proceeding.")
-def create_bb_worker(dirname, workername, workerpw):
+def create_bb_worker(systemd_unit, dirname, workername, workerpw):
home = Path.home()
bb_dir = home / dirname
if bb_dir.exists():
@@ -595,7 +609,40 @@ def create_bb_worker(dirname, workername, workerpw):
],
check=True,
)
+ setup_service (systemd_unit)
+
+def setup_service(systemd_unit):
+ sc_path = Path.home() / ".config" / "systemd" / "user"
+ sc_path.mkdir(exist_ok=True)
+ sc_unit = Path.home() / "deployment" / "systemd-services" / systemd_unit
+ copy(sc_unit, sc_path)
+ subprocess.run(
+ [
+ "systemctl",
+ "--user",
+ "daemon-reload",
+ ],
+ check=True,
+ )
+ subprocess.run(
+ [
+ "systemctl",
+ "--user",
+ "enable",
+ systemd_unit
+ ],
+ check=True,
+ )
+ subprocess.run(
+ [
+ "systemctl",
+ "--user",
+ "start",
+ systemd_unit
+ ],
+ check=True,
+ )
if __name__ == "__main__":
cli()
diff --git a/buildbot/build.sh b/buildbot/build.sh
@@ -2,14 +2,6 @@
set -eu
-PAUSE_LOCKFILE=/tmp/buildbot_pause.lock
-
-if [ -f $PAUSE_LOCKFILE ]; then
- echo "The Buildbot is paused, remove ${PAUSE_LOCKFILE} to resume."
- # Treat this as error, so dependent schedulers won't fire up.
- exit 1
-fi
-
echo "Running taler-deployment bootstrap"
source "${HOME}/activate"
taler-deployment bootstrap
diff --git a/buildbot/coverage.sh b/buildbot/coverage.sh
@@ -2,25 +2,31 @@
set -eu
-PAUSE_LOCKFILE=/tmp/buildbot_pause.lock
+source "${HOME}/activate"
-if [ -f $PAUSE_LOCKFILE ]; then
- echo "The Buildbot is paused, remove ${PAUSE_LOCKFILE} to resume."
- # Treat this as error, so dependent schedulers won't fire up.
- exit 1
-fi
+PG_VER=12
+PG_DIR=/usr/lib/postgresql/$PG_VER/bin
-source "${HOME}/activate"
+# setup little postgres DB for us ...
+TMP_DB_DIR=`mktemp tmpXXXXXX`
+$PG_DIR/initdb -D $TMP_DB_DIR/ -A trust
+$PG_DIR/createdb talercheck
+$PG_DIR/createdb synccheck
+$PG_DIR/createdb anastasischeck
+$PG_DIR/pg_ctl -D $TMP_DB_DIR/ -l logfile start
-for codebase in merchant exchange ; do
+for codebase in merchant exchange sync ; do
TOP="$HOME/sources/${codebase}/"
REPORT_DIR="$HOME/sources/${codebase}/coverage_report"
lcov -d $TOP -z
cd $TOP
- make check || true
+ make check -k || true
mkdir -p $REPORT_DIR
lcov -d $TOP -c --no-external -o $REPORT_DIR/coverage.info
lcov -r $REPORT_DIR/coverage.info **/test_* -o $REPORT_DIR/rcoverage.info
genhtml -o $REPORT_DIR $REPORT_DIR/rcoverage.info
chmod a+rx -R $REPORT_DIR
done
+
+$PG_DIR/pg_ctl -D $TMP_DB_DIR/ stop
+rm -rf $TMP_DB_DIR
diff --git a/envcfg.py.template b/envcfg.py.template
@@ -1,4 +1,4 @@
-# Name of the environment (test, demo, int, ...)
+# Name of the environment (test, demo, int, coverage, demo-checker, auditor-reporter, ...)
env = "..."
tag = "master"
diff --git a/systemd-services/buildbot-worker-auditor.service b/systemd-services/buildbot-worker-auditor.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Buildbot worker service for auditor
+AssertPathExists=%h/worker
+
+[Service]
+WorkingDirectory=%h
+ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
+ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
+ExecStop=/usr/local/bin/buildbot-worker stop worker
+Restart=always
+
+[Install]
+WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-demo-auditor.service b/systemd-services/buildbot-worker-demo-auditor.service
@@ -1,13 +0,0 @@
-[Unit]
-Description=Buildbot worker service for demo-auditor
-AssertPathExists=/home/demo-auditor/worker
-
-[Service]
-WorkingDirectory=/home/demo-auditor/
-ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
-ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
-ExecStop=/usr/local/bin/buildbot-worker stop worker
-Restart=always
-
-[Install]
-WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-taler-demo-healthcheck.service b/systemd-services/buildbot-worker-taler-demo-healthcheck.service
@@ -1,13 +0,0 @@
-[Unit]
-Description=Buildbot worker service for taler-demo-healthcheck
-AssertPathExists=/home/taler-demo-healthcheck/worker
-
-[Service]
-WorkingDirectory=/home/taler-demo-healthcheck/
-ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
-ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
-ExecStop=/usr/local/bin/buildbot-worker stop worker
-Restart=always
-
-[Install]
-WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-taler-healthcheck.service b/systemd-services/buildbot-worker-taler-healthcheck.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Buildbot worker service for taler-healthcheck
+AssertPathExists=%h/worker
+
+[Service]
+WorkingDirectory=%h
+ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
+ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
+ExecStop=/usr/local/bin/buildbot-worker stop worker
+Restart=always
+
+[Install]
+WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-taler-test-healthcheck.service b/systemd-services/buildbot-worker-taler-test-healthcheck.service
@@ -1,13 +0,0 @@
-[Unit]
-Description=Buildbot worker service for taler-test-healthcheck
-AssertPathExists=/home/taler-test-healthcheck/worker
-
-[Service]
-WorkingDirectory=/home/taler-test-healthcheck/
-ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
-ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
-ExecStop=/usr/local/bin/buildbot-worker stop worker
-Restart=always
-
-[Install]
-WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-taler.service b/systemd-services/buildbot-worker-taler.service
@@ -0,0 +1,13 @@
+[Unit]
+Description=Buildbot worker service for the main deployment (runs exchange, merchant, sync)
+AssertPathExists=%h/bb-worker
+
+[Service]
+WorkingDirectory=%h
+ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
+ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
+ExecStop=/usr/local/bin/buildbot-worker stop worker
+Restart=always
+
+[Install]
+WantedBy=default.target
diff --git a/systemd-services/buildbot-worker-test-auditor.service b/systemd-services/buildbot-worker-test-auditor.service
@@ -1,13 +0,0 @@
-[Unit]
-Description=Buildbot worker service for test-auditor
-AssertPathExists=/home/test-auditor/worker
-
-[Service]
-WorkingDirectory=/home/test-auditor/
-ExecStart=/usr/local/bin/buildbot-worker start --nodaemon worker
-ExecReload=/usr/local/bin/buildbot-worker restart --nodaemon worker
-ExecStop=/usr/local/bin/buildbot-worker stop worker
-Restart=always
-
-[Install]
-WantedBy=default.target
