#!/usr/bin/env python3 # This file is part of GNU Taler. # # GNU Taler is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # GNU Taler is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with GNU Taler. If not, see . import click import types import os import sys import os.path import subprocess import time from pathlib import Path from dataclasses import dataclass from typing import List, Callable from shutil import copy from taler_urls import get_urls activate_template = """\ #!/bin/bash # Generated by taler-deployment-bootstrap if ! echo $PATH | tr ":" '\\n' | grep "$HOME/deployment/bin" > /dev/null then export PATH="{curr_path}" fi export PYTHONUSERBASE=$HOME/local export TALER_BOOTSTRAP_TIMESTAMP={timestamp} export TALER_CONFIG_CURRENCY={currency} export TALER_ENV_NAME={envname} export TALER_ENV_URL_INTRO="{landing}" export TALER_ENV_URL_BANK="{bank}" export TALER_ENV_URL_MERCHANT_BLOG="{blog}" export TALER_ENV_URL_MERCHANT_DONATIONS="{donations}" export TALER_ENV_URL_MERCHANT_SURVEY="{survey}" export TALER_ENV_URL_AUDITOR="{auditor}" export TALER_ENV_URL_BACKOFFICE="{backoffice}" export TALER_ENV_URL_SYNC="{sync}" export TALER_ENV_MERCHANT_BACKEND="{merchant_backend}" export TALER_COVERAGE={coverage} """ @dataclass class Repo: name: str url: str deps: List[str] builder: Callable[["Repo", Path], None] class EnvInfo: def __init__(self, name, repos, cfg): self.name = name self.repos = [] for r in repos: tag = getattr(cfg, "tag_" + r.name.replace("-", "_")) # This check skips all the components that are # expected to be already installed; typically via # a distribution package manager. if not tag: continue self.repos.append(r) @click.group() def cli(): pass # map from environment name to currency currmap = { "test": "TESTKUDOS", "docs-builder": "TESTKUDOS", "coverage": "TESTKUDOS", "integrationtest": "TESTKUDOS", "demo": "KUDOS", "int": "INTKUDOS", "euro": "EUR", "chf": "CHF", "auditor-reporter-test": "TESTKUDOS", "auditor-reporter-demo": "KUDOS", "local": "LOCALKUDOS", "tanker": "SEK" } def update_checkout(r: Repo, p: Path): """Clean the repository's working directory and update it to the match the latest version of the upstream branch that we are tracking.""" subprocess.run(["git", "-C", str(p), "clean", "-fdx"], check=True) subprocess.run(["git", "-C", str(p), "fetch"], check=True) subprocess.run(["git", "-C", str(p), "reset"], check=True) res = subprocess.run( [ "git", "-C", str(p), "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}", ], stderr=subprocess.DEVNULL, stdout=subprocess.PIPE, encoding="utf-8", ) if res.returncode != 0: ref = "HEAD" else: ref = res.stdout.strip("\n ") print(f"resetting {r.name} to ref {ref}") subprocess.run(["git", "-C", str(p), "reset", "--hard", ref], check=True) def default_configure(*extra): pfx = Path.home() / "local" extra_list = list(extra) if int(os.environ.get("TALER_COVERAGE")): extra_list.append("--enable-coverage") subprocess.run(["./configure", f"--prefix={pfx}"] + extra_list, check=True) def pyconfigure(*extra): """For python programs, --prefix doesn't work.""" subprocess.run(["./configure"] + list(extra), check=True) def build_libmicrohttpd(r: Repo, p: Path): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) # Debian gnutls packages are too old ... default_configure("--with-gnutls=/usr/local") subprocess.run(["make"], check=True) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_gnunet(r: Repo, p: Path): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure( "--enable-logging=verbose", f"--with-microhttpd={pfx}", "--disable-documentation", ) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_exchange(r: Repo, p: Path): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure( "CFLAGS=-ggdb -O0", "--enable-logging=verbose", f"--with-microhttpd={pfx}", f"--with-gnunet={pfx}", ) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_wallet(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) default_configure() subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_twister(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure( "CFLAGS=-ggdb -O0", "--enable-logging=verbose", f"--with-exchange={pfx}", f"--with-gnunet={pfx}", ) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_merchant(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure( "CFLAGS=-ggdb -O0", "--enable-logging=verbose", f"--with-microhttpd={pfx}", f"--with-exchange={pfx}", f"--with-gnunet={pfx}", "--disable-doc", ) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_sync(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure( "CFLAGS=-ggdb -O0", "--enable-logging=verbose", f"--with-microhttpd={pfx}", f"--with-exchange={pfx}", f"--with-merchant={pfx}", f"--with-gnunet={pfx}", "--disable-doc", ) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_anastasis(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure( "CFLAGS=-ggdb -O0", "--enable-logging=verbose", f"--with-microhttpd={pfx}", f"--with-exchange={pfx}", f"--with-merchant={pfx}", f"--with-gnunet={pfx}", "--disable-doc", ) subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_bank(r, p): update_checkout(r, p) subprocess.run(["pip3", "install", "poetry"], check=True) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" pyconfigure() subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_demos(r, p): update_checkout(r, p) pfx = Path.home() / "local" pyconfigure() subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def build_backoffice(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"]) pfx = Path.home() / "local" default_configure() subprocess.run(["make", "build-single"]) (p / "taler-buildstamp").touch() def build_docs(r, p): update_checkout(r, p) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" subprocess.run(["make", "install"], check=True) (p / "taler-buildstamp").touch() def get_repos(envname): """Get a list of repos (topologically sorted) that should be build for the given environment""" print(f"Loading return repositories for {envname}.", file=sys.stderr) if envname in ("demochecker",): return [] if envname in ("docs-builder",): return [ Repo( "docs", "git://git.taler.net/docs", [], build_docs, ), Repo( "exchange", "git://git.taler.net/exchange", ["gnunet", "libmicrohttpd"], build_exchange, ), Repo( "merchant", "git://git.taler.net/merchant", ["exchange"], build_merchant, ), Repo( "sync", "git://git.taler.net/sync", ["exchange", "merchant"], build_sync, ), Repo( "anastasis", "git://git.taler.net/anastasis", ["exchange", "merchant"], build_anastasis, ), Repo( "wallet-core", "git://git.taler.net/wallet-core", [], build_wallet, ), ] if envname in ("coverage", "integrationtest",): return [ Repo( "libmicrohttpd", "git://git.gnunet.org/libmicrohttpd.git", [], build_libmicrohttpd, ), Repo( "gnunet", "git://git.gnunet.org/gnunet.git", [], build_gnunet), Repo( "exchange", "git://git.taler.net/exchange", ["gnunet", "libmicrohttpd"], build_exchange, ), Repo( "twister", "git://git.taler.net/twister", ["gnunet", "exchange"], build_twister, ), Repo( "merchant", "git://git.taler.net/merchant", ["exchange", "libmicrohttpd"], build_merchant, ), Repo( "sync", "git://git.taler.net/sync", ["exchange", "merchant"], build_sync, ), Repo( "anastasis", "git://git.taler.net/anastasis", ["exchange", "merchant"], build_anastasis, ), Repo( "bank", "git://git.taler.net/bank", [], build_bank ), Repo( "merchant-backoffice", "git://git.taler.net/merchant-backoffice", [], build_backoffice, ), ] # Note: these are currently not in use! if envname in ("euro", "chf"): return [ Repo( "libmicrohttpd", "git://git.gnunet.org/libmicrohttpd.git", [], build_libmicrohttpd, ), Repo( "gnunet", "git://git.gnunet.org/gnunet.git", [], build_gnunet, ), Repo( "exchange", "git://git.taler.net/exchange", ["gnunet", "libmicrohttpd"], build_exchange, ), Repo( "merchant", "git://git.taler.net/merchant", ["exchange", "libmicrohttpd"], build_merchant, ), Repo( "bank", "git://git.taler.net/bank", [], build_bank, ), Repo( "taler-merchant-demos", "git://git.taler.net/taler-merchant-demos", [], build_demos, ), ] if envname in ("tanker", "local", "demo", "int", "test", "auditor-reporter-test", "auditor-reporter-demo"): return [ Repo( "wallet-core", "git://git.taler.net/wallet-core", [], build_wallet, ), Repo( "libmicrohttpd", "git://git.gnunet.org/libmicrohttpd.git", [], build_libmicrohttpd, ), Repo( "gnunet", "git://git.gnunet.org/gnunet.git", [], build_gnunet, ), Repo( "exchange", "git://git.taler.net/exchange", ["gnunet", "libmicrohttpd"], build_exchange, ), Repo( "twister", "git://git.taler.net/twister", ["gnunet", "exchange"], build_twister, ), Repo( "merchant", "git://git.taler.net/merchant", ["exchange", "libmicrohttpd"], build_merchant, ), Repo( "sync", "git://git.taler.net/sync", ["exchange", "merchant", "libmicrohttpd"], build_sync, ), Repo( "bank", "git://git.taler.net/bank", [], build_bank, ), Repo( "taler-merchant-demos", "git://git.taler.net/taler-merchant-demos", [], build_demos, ), ] raise Exception(f"no repos defined for envname {envname}") def ensure_activated(): """Make sure that the environment variables have been loaded correctly via the ~/activate script""" ts = os.environ.get("TALER_BOOTSTRAP_TIMESTAMP") if ts is None: print("Please do 'source ~/activate' first.", file=sys.stderr) sys.exit(1) out = subprocess.check_output( ["bash", "-c", "source ~/activate; echo $TALER_BOOTSTRAP_TIMESTAMP"], encoding="utf-8", ) out = out.strip(" \n") if out != ts: print( f"Please do 'source ~/activate'. Current ts={ts}, new ts={out}", file=sys.stderr, ) sys.exit(1) def update_repos(repos: List[Repo]) -> None: for r in repos: r_dir = Path.home() / "sources" / r.name subprocess.run(["git", "-C", str(r_dir), "fetch"], check=True) res = subprocess.run( ["git", "-C", str(r_dir), "status", "-sb"], check=True, stdout=subprocess.PIPE, encoding="utf-8", ) if "behind" in res.stdout: print(f"new commits in {r}") s = r_dir / "taler-buildstamp" if s.exists(): s.unlink() def get_stale_repos(repos: List[Repo]) -> List[Repo]: timestamps = {} stale = [] for r in repos: r_dir = Path.home() / "sources" / r.name s = r_dir / "taler-buildstamp" if not s.exists(): timestamps[r.name] = time.time() stale.append(r) continue ts = timestamps[r.name] = s.stat().st_mtime for dep in r.deps: if timestamps[dep] > ts: stale.append(r) break return stale allowed_envs = ( "test", "int", "demo", "auditor-reporter-test", "auditor-reporter-demo", "docs-builder", "euro", "chf", "coverage", "integrationtest", "local", "tanker" ) def load_envcfg(): cfg = types.ModuleType("taler_deployment_cfg") envcfg_path = Path.home() / "envcfg.py" if not os.path.isfile(envcfg_path): return None print(f"Loading configuration from {envcfg_path}.", file=sys.stderr) cfgtext = envcfg_path.read_text() exec(cfgtext, cfg.__dict__) return cfg def get_env_info(cfg): envname = getattr(cfg, "env") if envname not in allowed_envs: print(f"env '{envname}' not supported") sys.exit(1) repos = get_repos(envname) return EnvInfo(envname, repos, cfg) @cli.command() def build() -> None: """Build the deployment from source.""" ensure_activated() cfg = load_envcfg() if not cfg: print("Please create ~/envcfg.py (template in deployment.git can help)") return 1 env_info = get_env_info(cfg) update_repos(env_info.repos) stale = get_stale_repos(env_info.repos) print(f"found stale repos: {stale}") for r in stale: p = Path.home() / "sources" / r.name os.chdir(str(p)) r.builder(r, p) @cli.command() @click.argument("color", metavar="COLOR", type=click.Choice(["blue", "green"])) def switch_demo(color) -> None: """Switch deployment color of demo.""" if os.environ["USER"] != "demo": print("Command should be executed as the demo user only.") sys.exit(1) active_home = Path.home() / "active-home" try: active_home.unlink() except: pass active_home.symlink_to(f"/home/demo-{color}") # repos does not contain distro-installed components def checkout_repos(cfg, repos): """Check out repos to the version specified in envcfg.py""" home = Path.home() sources = home / "sources" for r in repos: r_dir = home / "sources" / r.name if not r_dir.exists(): r_dir.mkdir(parents=True, exist_ok=True) subprocess.run(["git", "-C", str(sources), "clone", r.url], check=True) subprocess.run(["git", "-C", str(r_dir), "fetch"], check=True) tag = getattr(cfg, "tag_" + r.name.replace("-", "_")) subprocess.run( ["git", "-C", str(r_dir), "checkout", "-q", "-f", tag, "--"], check=True, ) @cli.command() def sync_repos() -> None: """Sync repos with the envcfg.py file.""" home = Path.home() cfg = load_envcfg() if not cfg: print("Please create ~/envcfg.py (template in deployment.git can help)") return 1 env_info = get_env_info(cfg) repos = env_info.repos checkout_repos(cfg, repos) for r in repos: r_dir = home / "sources" / r.name subprocess.run(["git", "-C", str(r_dir), "clean", "-fdx"], check=True) @cli.command() def bootstrap() -> None: """Bootstrap a GNU Taler deployment.""" home = Path.home() cfg = load_envcfg() if not cfg: print("Please create ~/envcfg.py (template in deployment.git can help)") return 1 env_info = get_env_info(cfg) repos = env_info.repos envname = env_info.name checkout_repos(cfg,repos) # Generate $PATH variable that will be set in the activate script. local_path = str(Path.home() / "local" / "bin") deployment_path = str(Path.home() / "deployment" / "bin") path_list = os.environ["PATH"].split(":") if local_path not in path_list: path_list.insert(0, local_path) if deployment_path not in path_list: path_list.insert(0, deployment_path) with (home / "activate").open("w") as f: f.write( activate_template.format( envname=envname, timestamp=str(time.time()), currency=currmap[envname], curr_path=":".join(path_list), coverage=1 if envname == "coverage" else 0, **get_urls(envname) ) ) if envname != "local": (home / "sockets").mkdir(parents=True, exist_ok=True) if envname in ("test", "int", "local"): (home / "taler-data").mkdir(parents=True, exist_ok=True) if envname == "demo": setup_service("config-tips.timer") if not (home / "taler-data").exists(): (home / "taler-data").symlink_to("/home/demo/shared-data") if envname == "test": create_bb_worker("buildbot-worker-taler.service", "bb-worker", "test-worker", "test-pass") setup_service("config-tips.timer") elif envname in ("auditor-reporter-test", "auditor-reporter-demo"): create_bb_worker("buildbot-worker-auditor.service", "worker", "auditor-worker", "auditor-pass") elif envname == "demo-checker": create_bb_worker("buildbot-worker-taler-healthcheck.service", "bb-worker", "demo-worker", "demo-pass") elif envname == "coverage": create_bb_worker("buildbot-worker-lcov.service", "worker", "lcov-worker", "lcov-pass") www_path = Path.home() / "www" www_path.mkdir(exist_ok=True) if not os.path.islink(www_path / "merchant"): os.symlink( Path.home() / "sources" / "merchant" / "coverage_report", www_path / "merchant", ) if not os.path.islink(www_path / "exchange"): os.symlink( Path.home() / "sources" / "exchange" / "coverage_report", www_path / "exchange", ) if not os.path.islink(www_path / "sync"): os.symlink( Path.home() / "sources" / "sync" / "coverage_report", www_path / "sync", ) print("Bootstrap finished.") print("Please source the ~/activate file before proceeding.") def create_bb_worker(systemd_unit, dirname, workername, workerpw): home = Path.home() bb_dir = home / dirname if bb_dir.exists(): return subprocess.run( [ "buildbot-worker", "create-worker", "--umask=0o22", str(bb_dir), "localhost:9989", workername, workerpw, ], check=True, ) setup_service (systemd_unit) def setup_service(systemd_unit): sc_path = Path.home() / ".config" / "systemd" / "user" sc_path.mkdir(exist_ok=True,parents=True) sc_unit = Path.home() / "deployment" / "systemd-services" / systemd_unit copy(sc_unit, sc_path) # If a timer got just installed, the related service # file needs to be installed now. split_filename = systemd_unit.split(".") if "timer" == split_filename[-1]: copy(Path.home() / "deployment" / "systemd-services" / f"{split_filename[0]}.service", sc_path) subprocess.run( [ "systemctl", "--user", "daemon-reload", ], check=True, ) subprocess.run( [ "systemctl", "--user", "enable", systemd_unit ], check=True, ) subprocess.run( [ "systemctl", "--user", "start", systemd_unit ], check=True, ) if __name__ == "__main__": cli()