diff options
author | MS <ms@taler.net> | 2022-08-25 15:38:41 +0200 |
---|---|---|
committer | MS <ms@taler.net> | 2022-08-25 15:38:41 +0200 |
commit | d5eacca1d5b2e37ba2f869568cc010aab4037502 (patch) | |
tree | 803662f32c02e14f2c0376801d96cae891b79f9d /bin/taler-deployment | |
parent | 022eef39974a6e32e63ba41db523d6ec1781d0e3 (diff) | |
download | deployment-d5eacca1d5b2e37ba2f869568cc010aab4037502.tar.gz deployment-d5eacca1d5b2e37ba2f869568cc010aab4037502.tar.bz2 deployment-d5eacca1d5b2e37ba2f869568cc010aab4037502.zip |
remove obsolete scripts
Diffstat (limited to 'bin/taler-deployment')
-rwxr-xr-x | bin/taler-deployment | 786 |
1 files changed, 0 insertions, 786 deletions
diff --git a/bin/taler-deployment b/bin/taler-deployment deleted file mode 100755 index 8a5ff9c..0000000 --- a/bin/taler-deployment +++ /dev/null @@ -1,786 +0,0 @@ -#!/usr/bin/env python3 - -# This file is part of GNU Taler. -# -# GNU Taler is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# GNU Taler is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with GNU Taler. If not, see <https://www.gnu.org/licenses/>. - -import click -import types -import os -import sys -import os.path -import subprocess -import time -import random -from pathlib import Path -from dataclasses import dataclass -from typing import List, Callable -from shutil import copy -from taler_urls import get_urls -from string import ascii_letters, ascii_uppercase - -activate_template = """\ -#!/bin/bash - -# Generated by taler-deployment-bootstrap - -if ! echo $PATH | tr ":" '\\n' | grep "$HOME/deployment/bin" > /dev/null -then - export PATH="{curr_path}" -fi -export PYTHONUSERBASE=$HOME/local -export TALER_BOOTSTRAP_TIMESTAMP={timestamp} -export TALER_CONFIG_CURRENCY={currency} -export TALER_ENV_NAME={envname} -export TALER_ENV_URL_INTRO="{landing}" -export TALER_ENV_URL_BANK="{bank}" -export TALER_ENV_URL_MERCHANT_BLOG="{blog}" -export TALER_ENV_URL_MERCHANT_DONATIONS="{donations}" -export TALER_ENV_URL_MERCHANT_SURVEY="{survey}" -export TALER_ENV_URL_AUDITOR="{auditor}" -export TALER_ENV_URL_BACKOFFICE="{backoffice}" -export TALER_ENV_URL_SYNC="{sync}" -export TALER_ENV_MERCHANT_BACKEND="{merchant_backend}" -export TALER_COVERAGE={coverage} -export TALER_ENV_FRONTENDS_APITOKEN="$(cat ~/merchant_auth_token)" -export LIBEUFIN_ENV_SANDBOX_ADMIN_PASSWORD="$(cat ~/libeufin_admin_password)" -export LIBEUFIN_NEXUS_DB_CONNECTION="jdbc:sqlite:$HOME/nexus.sqlite" -export LIBEUFIN_SANDBOX_DB_CONNECTION="jdbc:sqlite:$HOME/sandbox.sqlite" -export LIBEUFIN_SANDBOX_HOSTNAME=bank.{envname}.taler.net/eufin/sandbox -export LIBEUFIN_SANDBOX_CURRENCY={currency} -""" - -@dataclass -class Repo: - name: str - url: str - deps: List[str] - builder: Callable[["Repo", Path], None] - - -class EnvInfo: - def __init__(self, name, repos, cfg): - self.name = name - self.repos = [] - for r in repos: - tag = getattr(cfg, "tag_" + r.name.replace("-", "_")) - # This check skips all the components that are - # expected to be already installed; typically via - # a distribution package manager. - if not tag: - continue - self.repos.append(r) - -@click.group() -def cli(): - pass - - -# map from environment name to currency -currmap = { - "test": "TESTKUDOS", - "docs-builder": "TESTKUDOS", - "coverage": "TESTKUDOS", - "integrationtest": "TESTKUDOS", - "demo": "KUDOS", - "int": "INT", - "euro": "EUR", - "chf": "CHF", - "auditor-reporter-test": "TESTKUDOS", - "auditor-reporter-demo": "KUDOS", - "local": "LOCALKUDOS", - "tanker": "SEK" -} - -def generate_apitoken(): - return "secret-token:" + ''.join(random.choices(ascii_letters + ascii_uppercase, k=10)) - -def generate_password(): - return ''.join(random.choices(ascii_letters + ascii_uppercase, k=10)) - - -def update_checkout(r: Repo, p: Path): - """Clean the repository's working directory and - update it to the match the latest version of the upstream branch - that we are tracking.""" - subprocess.run(["git", "-C", str(p), "clean", "-fdx"], check=True) - subprocess.run(["git", "-C", str(p), "fetch"], check=True) - subprocess.run(["git", "-C", str(p), "reset"], check=True) - res = subprocess.run( - [ - "git", - "-C", - str(p), - "rev-parse", - "--abbrev-ref", - "--symbolic-full-name", - "@{u}", - ], - stderr=subprocess.DEVNULL, - stdout=subprocess.PIPE, - encoding="utf-8", - ) - if res.returncode != 0: - ref = "HEAD" - else: - ref = res.stdout.strip("\n ") - print(f"resetting {r.name} to ref {ref}") - subprocess.run(["git", "-C", str(p), "reset", "--hard", ref], check=True) - - -def default_configure(*extra): - pfx = Path.home() / "local" - extra_list = list(extra) - if int(os.environ.get("TALER_COVERAGE")): - extra_list.append("--enable-coverage") - subprocess.run(["./configure", f"--prefix={pfx}"] + extra_list, check=True) - -def default_configure_nc(*extra): - """Variant of default_configure() that does NEVER add --enable-coverage""" - pfx = Path.home() / "local" - extra_list = list(extra) - subprocess.run(["./configure", f"--prefix={pfx}"] + extra_list, check=True) - - -def pyconfigure(*extra): - """For python programs, --prefix doesn't work.""" - subprocess.run(["./configure"] + list(extra), check=True) - - -def build_libeufin(r: Repo, p: Path): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - default_configure_nc() - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - -def build_libmicrohttpd(r: Repo, p: Path): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - # Debian gnutls packages are too old ... - default_configure("--with-gnutls=/usr/local") - subprocess.run(["make"], check=True) - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - - -def build_gnunet(r: Repo, p: Path): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - pfx = Path.home() / "local" - default_configure( - "--enable-logging=verbose", - f"--with-microhttpd={pfx}", - "--disable-documentation", - ) - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - - -def build_exchange(r: Repo, p: Path): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - pfx = Path.home() / "local" - default_configure( - "CFLAGS=-ggdb -O0", - "--enable-logging=verbose", - f"--with-microhttpd={pfx}", - f"--with-gnunet={pfx}", - ) - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - - -def build_wallet(r, p): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - default_configure_nc() - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - -def build_twister(r, p): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - pfx = Path.home() / "local" - default_configure( - "CFLAGS=-ggdb -O0", - "--enable-logging=verbose", - f"--with-exchange={pfx}", - f"--with-gnunet={pfx}", - ) - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - - -def build_merchant(r, p): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - pfx = Path.home() / "local" - default_configure( - "CFLAGS=-ggdb -O0", - "--enable-logging=verbose", - f"--with-microhttpd={pfx}", - f"--with-exchange={pfx}", - f"--with-gnunet={pfx}", - "--disable-doc", - ) - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - -def build_sync(r, p): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - pfx = Path.home() / "local" - default_configure( - "CFLAGS=-ggdb -O0", - "--enable-logging=verbose", - f"--with-microhttpd={pfx}", - f"--with-exchange={pfx}", - f"--with-merchant={pfx}", - f"--with-gnunet={pfx}", - "--disable-doc", - ) - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - - -def build_demos(r, p): - update_checkout(r, p) - pfx = Path.home() / "local" - pyconfigure() - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - -def build_backoffice(r, p): - update_checkout(r, p) - subprocess.run(["./bootstrap"]) - subprocess.run(["./configure"]) - subprocess.run(["make", "build-single"]) - (p / "taler-buildstamp").touch() - -def build_docs(r, p): - update_checkout(r, p) - subprocess.run(["./bootstrap"], check=True) - pfx = Path.home() / "local" - subprocess.run(["make", "install"], check=True) - (p / "taler-buildstamp").touch() - - -def get_repos(envname): - """Get a list of repos (topologically sorted) that should be build for the - given environment""" - print(f"Loading return repositories for {envname}.", file=sys.stderr) - if envname in ("demochecker",): - return [] - if envname in ("docs-builder",): - return [ - Repo( - "libeufin", - "git://git.taler.net/libeufin.git", - [], - build_libeufin, - ), - Repo( - "libmicrohttpd", - "git://git.gnunet.org/libmicrohttpd.git", - [], - build_libmicrohttpd, - ), - Repo( - "docs", - "git://git.taler.net/docs", - [], - build_docs, - ), - Repo( - "wallet-core", - "git://git.taler.net/wallet-core", - [], - build_wallet, - ), - Repo( - "exchange", - "git://git.taler.net/exchange", - ["gnunet", "libmicrohttpd"], - build_exchange, - ), - Repo( - "merchant", - "git://git.taler.net/merchant", - ["exchange","libmicrohttpd","gnunet"], - build_merchant, - ), - Repo( - "sync", - "git://git.taler.net/sync", - ["exchange", "merchant","gnunet","libmicrohttpd"], - build_sync, - ), - ] - if envname in ("int", "coverage", "integrationtest",): - return [ - Repo( - "libeufin", - "git://git.taler.net/libeufin.git", - [], - build_libeufin, - ), - Repo( - "libmicrohttpd", - "git://git.gnunet.org/libmicrohttpd.git", - [], - build_libmicrohttpd, - ), - Repo( - "gnunet", - "git://git.gnunet.org/gnunet.git", - ["libmicrohttpd"], - build_gnunet), - Repo( - "wallet-core", - "git://git.taler.net/wallet-core", - [], - build_wallet, - ), - Repo( - "exchange", - "git://git.taler.net/exchange", - ["gnunet", "libmicrohttpd", "wallet-core"], - build_exchange, - ), - Repo( - "merchant", - "git://git.taler.net/merchant", - ["exchange", "libmicrohttpd", "gnunet"], - build_merchant, - ), - Repo( - "sync", - "git://git.taler.net/sync", - ["exchange", "merchant", "gnunet", "libmicrohttpd"], - build_sync, - ), - ] - - # Note: these are currently not in use! - if envname in ("euro", "chf"): - return [ - Repo( - "libeufin", - "git://git.taler.net/libeufin.git", - [], - build_libeufin, - ), - Repo( - "libmicrohttpd", - "git://git.gnunet.org/libmicrohttpd.git", - [], - build_libmicrohttpd, - ), - Repo( - "gnunet", - "git://git.gnunet.org/gnunet.git", - ["libmicrohttpd"], - build_gnunet, - ), - Repo( - "exchange", - "git://git.taler.net/exchange", - ["gnunet", "libmicrohttpd"], - build_exchange, - ), - Repo( - "merchant", - "git://git.taler.net/merchant", - ["exchange", "libmicrohttpd"], - build_merchant, - ), - Repo( - "taler-merchant-demos", - "git://git.taler.net/taler-merchant-demos", - [], - build_demos, - ), - ] - if envname in ("tanker", "local", "demo", "int", "test", "auditor-reporter-test", "auditor-reporter-demo"): - return [ - Repo( - "libeufin", - "git://git.taler.net/libeufin.git", - [], - build_libeufin, - ), - Repo( - "wallet-core", - "git://git.taler.net/wallet-core", - [], - build_wallet, - ), - Repo( - "libmicrohttpd", - "git://git.gnunet.org/libmicrohttpd.git", - [], - build_libmicrohttpd, - ), - Repo( - "gnunet", - "git://git.gnunet.org/gnunet.git", - ["libmicrohttpd"], - build_gnunet, - ), - Repo( - "twister", - "git://git.taler.net/twister", - ["gnunet", "libmicrohttpd"], - build_twister, - ), - Repo( - "exchange", - "git://git.taler.net/exchange", - ["gnunet", "libmicrohttpd", "twister", "wallet-core"], - build_exchange, - ), - Repo( - "merchant", - "git://git.taler.net/merchant", - ["exchange", "libmicrohttpd", "gnunet"], - build_merchant, - ), - Repo( - "sync", - "git://git.taler.net/sync", - ["exchange", "merchant", "gnunet", "libmicrohttpd"], - build_sync, - ), - Repo( - "taler-merchant-demos", - "git://git.taler.net/taler-merchant-demos", - [], - build_demos, - ), - ] - raise Exception(f"no repos defined for envname {envname}") - - -def ensure_activated(): - """Make sure that the environment variables have been - loaded correctly via the ~/activate script""" - ts = os.environ.get("TALER_BOOTSTRAP_TIMESTAMP") - if ts is None: - print("Please do 'source ~/activate' first.", file=sys.stderr) - sys.exit(1) - out = subprocess.check_output( - ["bash", "-c", "source ~/activate; echo $TALER_BOOTSTRAP_TIMESTAMP"], - encoding="utf-8", - ) - out = out.strip(" \n") - if out != ts: - print( - f"Please do 'source ~/activate'. Current ts={ts}, new ts={out}", - file=sys.stderr, - ) - sys.exit(1) - - -def update_repos(cfg, repos: List[Repo]) -> None: - for r in repos: - tag = getattr(cfg, "tag_" + r.name.replace("-", "_")) - if tag == "SKIP": - continue - r_dir = Path.home() / "sources" / r.name - subprocess.run(["git", "-C", str(r_dir), "fetch"], check=True) - res = subprocess.run( - ["git", "-C", str(r_dir), "status", "-sb"], - check=True, - stdout=subprocess.PIPE, - encoding="utf-8", - ) - if "behind" in res.stdout: - print(f"new commits in {r}") - s = r_dir / "taler-buildstamp" - if s.exists(): - s.unlink() - - -def get_stale_repos(repos: List[Repo]) -> List[Repo]: - timestamps = {} - stale = [] - for r in repos: - r_dir = Path.home() / "sources" / r.name - s = r_dir / "taler-buildstamp" - if not s.exists(): - timestamps[r.name] = time.time() - stale.append(r) - continue - ts = timestamps[r.name] = s.stat().st_mtime - for dep in r.deps: - if timestamps[dep] > ts: - stale.append(r) - break - return stale - - -allowed_envs = ( - "test", - "int", - "demo", - "auditor-reporter-test", - "auditor-reporter-demo", - "docs-builder", - "euro", - "chf", - "coverage", - "integrationtest", - "local", - "tanker" -) - -def load_apitoken(): - apitoken_path = Path.home() / "envcfg.py" - if not os.path.isfile(apitoken_path): - return None - with open(apitoken_path, "r") as f: - return f.readline() - -def load_envcfg(): - cfg = types.ModuleType("taler_deployment_cfg") - envcfg_path = Path.home() / "envcfg.py" - if not os.path.isfile(envcfg_path): - return None - print(f"Loading configuration from {envcfg_path}.", file=sys.stderr) - cfgtext = envcfg_path.read_text() - exec(cfgtext, cfg.__dict__) - return cfg - - -def get_env_info(cfg): - envname = getattr(cfg, "env") - if envname not in allowed_envs: - print(f"env '{envname}' not supported") - sys.exit(1) - repos = get_repos(envname) - return EnvInfo(envname, repos, cfg) - - -@cli.command() -def build() -> None: - """Build the deployment from source.""" - ensure_activated() - cfg = load_envcfg() - if not cfg: - print("Please create ~/envcfg.py (template in deployment.git can help)") - return 1 - env_info = get_env_info(cfg) - update_repos(cfg, env_info.repos) - stale = get_stale_repos(env_info.repos) - print(f"found stale repos: {stale}") - for r in stale: - tag = getattr(cfg, "tag_" + r.name.replace("-", "_")) - if tag == "SKIP": - continue - p = Path.home() / "sources" / r.name - os.chdir(str(p)) - r.builder(r, p) - - -# repos does not contain distro-installed components -def checkout_repos(cfg, repos): - """Check out repos to the version specified in envcfg.py""" - home = Path.home() - sources = home / "sources" - for r in repos: - r_dir = home / "sources" / r.name - tag = getattr(cfg, "tag_" + r.name.replace("-", "_")) - if tag == "SKIP": - continue - if not r_dir.exists(): - r_dir.mkdir(parents=True, exist_ok=True) - subprocess.run(["git", "-C", str(sources), "clone", r.url], check=True) - subprocess.run(["git", "-C", str(r_dir), "fetch"], check=True) - subprocess.run( - ["git", "-C", str(r_dir), "checkout", "-q", "-f", tag, "--"], check=True, - ) - - -@cli.command() -def sync_repos() -> None: - """Sync repos with the envcfg.py file.""" - home = Path.home() - cfg = load_envcfg() - if not cfg: - print("Please create ~/envcfg.py (template in deployment.git can help)") - return 1 - env_info = get_env_info(cfg) - repos = env_info.repos - checkout_repos(cfg, repos) - for r in repos: - r_dir = home / "sources" / r.name - subprocess.run(["git", "-C", str(r_dir), "clean", "-fdx"], check=True) - -@cli.command() -def bootstrap() -> None: - """Bootstrap a GNU Taler deployment.""" - home = Path.home() - cfg = load_envcfg() - if not cfg: - print("Please create ~/envcfg.py (template in deployment.git can help)") - return 1 - env_info = get_env_info(cfg) - repos = env_info.repos - envname = env_info.name - checkout_repos(cfg,repos) - - # Generate $PATH variable that will be set in the activate script. - local_path = str(Path.home() / "local" / "bin") - deployment_path = str(Path.home() / "deployment" / "bin") - path_list = os.environ["PATH"].split(":") - if local_path not in path_list: - path_list.insert(0, local_path) - if deployment_path not in path_list: - path_list.insert(0, deployment_path) - - token_file = Path.home() / "merchant_auth_token" - if not token_file.is_file(): - with token_file.open("w") as f: - f.write(generate_apitoken()) - print(f"Token file '{token_file}' created.") - - sandbox_admin_password_file = Path.home() / "libeufin_admin_password" - if not sandbox_admin_password_file.is_file(): - with sandbox_admin_password_file.open("w") as f: - f.write(generate_password()) - print(f"Libeufin Sandbox admin password file '{sandbox_admin_password_file}' created.") - - with (home / "activate").open("w") as f: - f.write( - activate_template.format( - envname=envname, - timestamp=str(time.time()), - currency=currmap[envname], - curr_path=":".join(path_list), - coverage=1 if envname == "coverage" else 0, - **get_urls(envname) - ) - ) - if envname != "local": - (home / "sockets").mkdir(parents=True, exist_ok=True) - - if envname in ("test", "int", "local"): - (home / "taler-data").mkdir(parents=True, exist_ok=True) - if envname == "demo": - setup_service("config-tips.timer") - create_bb_worker( - "tips-checker.service", "tips-checker-dir", - "tips-checker-worker", "tips-checker-pass" - ) - - if not (home / "taler-data").exists(): - (home / "taler-data").symlink_to("/home/demo/shared-data") - - if envname == "integrationtest": - create_bb_worker("buildbot-worker-wallet.service", "worker", "wallet-worker", "wallet-pass") - - if envname == "test": - create_bb_worker("buildbot-worker-taler.service", "bb-worker", "test-worker", "test-pass") - setup_service("config-tips.timer") - - elif envname in ("auditor-reporter-test", "auditor-reporter-demo"): - create_bb_worker("buildbot-worker-auditor.service", "worker", "auditor-worker", "auditor-pass") - elif envname == "demo-checker": - create_bb_worker("buildbot-worker-taler-healthcheck.service", "bb-worker", "demo-worker", "demo-pass") - elif envname == "coverage": - create_bb_worker("buildbot-worker-lcov.service", "worker", "lcov-worker", "lcov-pass") - - www_path = Path.home() / "www" - www_path.mkdir(exist_ok=True) - if not os.path.islink(www_path / "merchant"): - os.symlink( - Path.home() / "sources" / "merchant" / "coverage_report", - www_path / "merchant", - ) - if not os.path.islink(www_path / "exchange"): - os.symlink( - Path.home() / "sources" / "exchange" / "coverage_report", - www_path / "exchange", - ) - if not os.path.islink(www_path / "sync"): - os.symlink( - Path.home() / "sources" / "sync" / "coverage_report", - www_path / "sync", - ) - - print("Bootstrap finished.") - print("Please source the ~/activate file before proceeding.") - - -def create_bb_worker(systemd_unit, dirname, workername, workerpw): - home = Path.home() - bb_dir = home / dirname - if bb_dir.exists(): - return - subprocess.run( - [ - "buildbot-worker", - "create-worker", - "--umask=0o22", - str(bb_dir), - "localhost:9989", - workername, - workerpw, - ], - check=True, - ) - setup_service (systemd_unit) - -def setup_service(systemd_unit): - sc_path = Path.home() / ".config" / "systemd" / "user" - sc_path.mkdir(exist_ok=True,parents=True) - sc_unit = Path.home() / "deployment" / "systemd-services" / systemd_unit - copy(sc_unit, sc_path) - - # If a timer got just installed, the related service - # file needs to be installed now. - split_filename = systemd_unit.split(".") - if "timer" == split_filename[-1]: - copy(Path.home() / "deployment" / "systemd-services" / f"{split_filename[0]}.service", sc_path) - - subprocess.run( - [ - "systemctl", - "--user", - "daemon-reload", - ], - check=True, - ) - subprocess.run( - [ - "systemctl", - "--user", - "enable", - systemd_unit - ], - check=True, - ) - subprocess.run( - [ - "systemctl", - "--user", - "start", - systemd_unit - ], - check=True, - ) - -if __name__ == "__main__": - cli() |