From 05f3689e3e4f3af98a4c720b6eb496d1934dc614 Mon Sep 17 00:00:00 2001 From: Florian Dold Date: Fri, 1 Nov 2019 11:39:40 +0100 Subject: taler-deployment --- bin/taler-deployment | 178 ++++++++++++++++++++++++++++++--------------------- envconfig.template | 13 ---- 2 files changed, 104 insertions(+), 87 deletions(-) delete mode 100644 envconfig.template diff --git a/bin/taler-deployment b/bin/taler-deployment index b06d064..a966050 100755 --- a/bin/taler-deployment +++ b/bin/taler-deployment @@ -63,6 +63,15 @@ class Repo: builder: Callable[["Repo", Path], None] +@dataclass +class EnvInfo: + name: str + repos: List[Repo] + + +pass_env_info = click.make_pass_decorator(EnvInfo) + + @click.group() def cli(): pass @@ -107,12 +116,10 @@ def default_configure(*extra): ["./configure", f"--prefix={pfx.as_posix()}"] + list(extra), check=True ) + def pyconfigure(*extra): """For python programs, --prefix doesn't work.""" - subprocess.run( - ["./configure", "--destination=local"] + list(extra), check=True - ) - + subprocess.run(["./configure", "--destination=local"] + list(extra), check=True) def build_libmicrohttpd(r: Repo, p: Path): @@ -196,7 +203,6 @@ def build_bank(r, p): def build_landing(r, p): update_checkout(r, p) - subprocess.run(["pip3", "install", "--user", "jsmin"], check=True) subprocess.run(["./bootstrap"], check=True) pfx = Path.home() / "local" default_configure() @@ -240,57 +246,66 @@ def build_backoffice(r, p): (p / "taler-buildstamp").touch() -# Our repositories, *must* be topologically sorted -repos = [ - Repo( - "libmicrohttpd", "git://gnunet.org/libmicrohttpd.git", [], build_libmicrohttpd - ), - Repo("gnunet", "git://gnunet.org/gnunet.git", [], build_gnunet), - Repo( - "exchange", - "git://git.taler.net/exchange", - ["gnunet", "libmicrohttpd"], - build_exchange, - ), - Repo( - "twister", "git://git.taler.net/twister", ["gnunet", "exchange"], build_twister - ), - Repo( - "merchant", - "git://git.taler.net/merchant", - ["exchange", "libmicrohttpd"], - build_merchant, - ), - Repo("bank", "git://git.taler.net/bank", [], build_bank), - Repo("landing", "git://git.taler.net/landing", [], build_landing), - Repo("donations", "git://git.taler.net/donations", [], build_donations), - Repo("blog", "git://git.taler.net/blog", [], build_blog), - Repo("survey", "git://git.taler.net/survey", [], build_survey), - Repo("backoffice", "git://git.taler.net/backoffice", [], build_backoffice), -] - - -def ensure_activated(): - """Make sure that the environment variables have been - loaded correctly via the ~/activate script""" - ts = os.environ.get("TALER_BOOTSTRAP_TIMESTAMP") - if ts is None: - print("Please do 'source ~/activate' first.", file=sys.stderr) - sys.exit(1) - out = subprocess.check_output( - ["bash", "-c", "source ~/activate; echo $TALER_BOOTSTRAP_TIMESTAMP"], - encoding="utf-8", - ) - out = out.strip(" \n") - if out != ts: - print( - f"Please do 'source ~/activate'. Current ts={ts}, new ts={out}", - file=sys.stderr, - ) - sys.exit(1) +def build_docs(r, p): + update_checkout(r, p) + subprocess.run(["./bootstrap"], check=True) + pfx = Path.home() / "local" + subprocess.run(["make", "install"], check=True) + (p / "taler-buildstamp").touch() -def update_repos(): +def get_repos(envname): + """Get a list of repos (topologically sorted) that should be build for the + given environment""" + if envname in ("demochecker",): + return [] + if envname == "docs-builder": + return [ + Repo( + "docs", + "git://gnunet.org/docs.git", + [], + build_docs, + ), + ] + if envname in ("demo", "int", "test", "auditor-reporter"): + return [ + Repo( + "libmicrohttpd", + "git://gnunet.org/libmicrohttpd.git", + [], + build_libmicrohttpd, + ), + Repo("gnunet", "git://gnunet.org/gnunet.git", [], build_gnunet), + Repo( + "exchange", + "git://git.taler.net/exchange", + ["gnunet", "libmicrohttpd"], + build_exchange, + ), + Repo( + "twister", + "git://git.taler.net/twister", + ["gnunet", "exchange"], + build_twister, + ), + Repo( + "merchant", + "git://git.taler.net/merchant", + ["exchange", "libmicrohttpd"], + build_merchant, + ), + Repo("bank", "git://git.taler.net/bank", [], build_bank), + Repo("landing", "git://git.taler.net/landing", [], build_landing), + Repo("donations", "git://git.taler.net/donations", [], build_donations), + Repo("blog", "git://git.taler.net/blog", [], build_blog), + Repo("survey", "git://git.taler.net/survey", [], build_survey), + Repo("backoffice", "git://git.taler.net/backoffice", [], build_backoffice), + ] + raise Exception(f"no repos defined for envname {envname}") + + +def update_repos(repos: List[Repo]) -> None: for r in repos: r_dir = Path.home() / "sources" / r.name subprocess.run(["git", "-C", r_dir.as_posix(), "fetch"], check=True) @@ -307,7 +322,7 @@ def update_repos(): s.unlink() -def get_stale_repos() -> List[Repo]: +def get_stale_repos(repos: List[Repo]) -> List[Repo]: timestamps = {} stale = [] for r in repos: @@ -326,25 +341,30 @@ def get_stale_repos() -> List[Repo]: @cli.command() -def build() -> None: +@pass_env_info +def build(env_info) -> None: """Build the deployment from source.""" ensure_activated() - update_repos() - stale = get_stale_repos() + update_repos(env_info.repos) + stale = get_stale_repos(env_info.repos) print(f"found stale repos: {stale}") for r in stale: p = Path.home() / "sources" / r.name os.chdir(p.as_posix()) r.builder(r, p) # type: ignore +allowed_envs = ("test", "int", "demo", "auditor-reporter", "docs-builder") @cli.command() -@click.argument("envname", type=click.Choice(["test", "int", "demo"])) -def bootstrap(envname) -> None: +@click.argument("envname", type=click.Choice(allowed_envs)) +@click.pass_context +def bootstrap(ctx, envname) -> None: """Bootstrap a GNU Taler deployment.""" home = Path.home() + ctx.env_info = EnvInfo(envname, get_repos(envname)) + # read Python-style config cfg = types.ModuleType("taler_deployment_cfg") cfgtext = (home / "envcfg.py").read_text() @@ -380,23 +400,33 @@ def bootstrap(envname) -> None: (home / "taler-data").mkdir(parents=True, exist_ok=True) if envname == "test": - bb_dir = home / "bb-worker" - if not bb_dir.exists(): - subprocess.run( - [ - "buildbot-worker", - "create-worker", - "--umask=0o22", - bb_dir.as_posix(), - "localhost:9989", - "test-worker", - "test-pass", - ], - check=True, - ) + create_bb_worker("bb-worker", "test-worker", "test-pass") + elif envname == "auditor-reporter": + create_bb_worker("bb-worker", "auditor-worker", "auditor-pass") + elif envname == "demo-checker": + create_bb_worker("bb-worker", "demo-worker", "demo-pass") + print("Bootstrap finished.") print("Please source the ~/activate file before proceeding.") +def create_bb_worker(dirname, workername, workerpw): + bb_dir = home / dirname + if bb_dir.exists(): + return + subprocess.run( + [ + "buildbot-worker", + "create-worker", + "--umask=0o22", + bb_dir.as_posix(), + "localhost:9989", + workername, + workerpw, + ], + check=True, + ) + + if __name__ == "__main__": cli() diff --git a/envconfig.template b/envconfig.template deleted file mode 100644 index 8c41494..0000000 --- a/envconfig.template +++ /dev/null @@ -1,13 +0,0 @@ -# Template for the environment configuration - -tag=demo-2019-08-31-00 - -tag_exchange=$tag -tag_merchant=$tag -tag_bank=$tag -tag_twister=$tag -tag_landing=$tag -tag_donations=$tag -tag_blog=$tag -tag_survey=$tag -tag_backoffice=$tag -- cgit v1.2.3