taler-deployment

Deployment scripts and configuration files
Log | Files | Refs | README

commit fb1228ac018ef8295f403acc327ce58dcd535fbf
parent 17d783c133089b6212a72448daf4d1e0989d8106
Author: Christian Grothoff <christian@grothoff.org>
Date:   Tue, 10 Nov 2026 01:53:46 +0100

buildbot: run nightly-only CI jobs from a nightly scheduler

Stages are discovered by listing contrib/ci/jobs, which cannot express "not on
every push", so a job now declares RUN_ON in its config.ini and the stage skips
itself in the builds it is not for. taler-android's three-hour emulator suite is
the first to need it; onlyIfChanged keeps a day with no pushes quiet.

Assisted by LLM

Diffstat:
Mbuildbot/master.cfg | 73++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 70 insertions(+), 3 deletions(-)

diff --git a/buildbot/master.cfg b/buildbot/master.cfg @@ -216,7 +216,8 @@ def container_add_step(HALT_ON_FAILURE, stepName, CONTAINER_ARCH="amd64", jobCmd="/workdir/contrib/ci/ci.sh", - containerFile="contrib/ci/Containerfile"): + containerFile="contrib/ci/Containerfile", + RUN_ON="push"): print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") # Locally built images belong to this repository, job and architecture. @@ -271,6 +272,29 @@ def container_add_step(HALT_ON_FAILURE, "-f", containerFile, "."], logname='build container', haltOnFailure=True) + # Stages come from `ls contrib/ci/jobs`, which cannot express "not in this + # kind of build". So each job says which schedulers it is for and the + # stage skips itself in the others. Without this a nightly would re-run + # the deployment jobs and publish a second set of nightly APKs, and a push + # would wait for a job meant to run once a day. + # + # `scheduler` is set by buildbot on every build a scheduler triggers, so + # the two names below are the two automatic cases and everything else is + # somebody asking for a build by hand. + def runs_in_this_build(step): + scheduler = step.getProperty("scheduler", "") or "" + if scheduler == f"{repoName}-nightly": + return RUN_ON in ("nightly", "both") + if scheduler == f"{repoName}-container-scheduler": + return RUN_ON in ("push", "both") + # Anything else -- a forced build, or a repository wired up by hand + # rather than through the container loop -- runs every stage. It was + # asked for explicitly, and a nightly-only job that could not be + # forced would be a job nobody could try out before tomorrow. + return True + + skipped = lambda results, s: results == SKIPPED + if not CONTAINER_BUILD: return steps.ShellSequence( name=stepName, @@ -278,6 +302,8 @@ def container_add_step(HALT_ON_FAILURE, haltOnFailure=HALT_ON_FAILURE, flunkOnFailure=True, warnOnWarnings=True, + doStepIf=runs_in_this_build, + hideStepIf=skipped, workdir=WORK_DIR ) else: @@ -287,6 +313,8 @@ def container_add_step(HALT_ON_FAILURE, haltOnFailure=HALT_ON_FAILURE, flunkOnFailure=True, warnOnWarnings=True, + doStepIf=runs_in_this_build, + hideStepIf=skipped, workdir=WORK_DIR ) @@ -942,7 +970,14 @@ def default_job_config(jobName, repoName): "WARN_ON_FAILURE": False, "CONTAINER_BUILD": True, "CONTAINER_NAME": repoName, - "CONTAINER_ARCH": "amd64"}} + "CONTAINER_ARCH": "amd64", + # Which schedulers a job is for: "push", "nightly" or + # "both". The default keeps every existing job exactly + # where it was -- stages are discovered by listing + # contrib/ci/jobs, so a job that said nothing used to + # run in every build of its repository, and a repository + # without a nightly scheduler still has only one kind. + "RUN_ON": "push"}} # Parse a job config and apply it as overrides to the standard configuration. @@ -1036,7 +1071,8 @@ class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): self.REPO_NAME, stage, self.configDict[stage]["CONTAINER_ARCH"], - f"contrib/ci/jobs/{stage}/job.sh") + f"contrib/ci/jobs/{stage}/job.sh", + RUN_ON=self.configDict[stage]["RUN_ON"]) for stage in jobDirs ]) @@ -1094,6 +1130,12 @@ container_repos = ["git.gnunet.org/gnunet", "git.taler.net/taldir", "git.taler.net/taler-typescript-core",] +# Repositories with at least one `RUN_ON = nightly` job in contrib/ci/jobs. +# Listed rather than inferred because the stage list is only known once a +# build is running, while a scheduler has to exist before that: a nightly for +# a repository without such a job would be a build whose every stage skips. +nightly_container_repos = ["taler-android"] + for repo in container_repos: # Factory-wide variables @@ -1184,6 +1226,31 @@ for repo in container_repos: builderNames=[f"{REPO_NAME}-builder"] )) + # Jobs whose config.ini says `RUN_ON = nightly` run from here instead of + # on every push: the android integration suite drives a real emulator and + # takes about three hours, which is not something to put in front of the + # next push to a worker shared with every other repository. + # + # onlyIfChanged keeps a quiet day quiet -- no change to this repository + # since the last build means no build tonight. Note that changes arrive + # here from the git hook (PBChangeSource), not from a poller, so a + # repository whose hook is silent gets no nightly rather than a redundant + # one. + # + # The builder is the same one the push scheduler uses; what differs is the + # `scheduler` property, which is how container_add_step decides whether a + # given stage belongs in this build. + if REPO_NAME in nightly_container_repos: + SCHEDULERS.append(schedulers.Nightly( + name=f"{REPO_NAME}-nightly", + change_filter=util.ChangeFilter( + branch="master", + project_re=f"^({REPO_NAME})$" + ), + builderNames=[f"{REPO_NAME}-builder"], + onlyIfChanged=True, + hour=2, minute=30)) + SERVICES.append(reporters.MailNotifier( fromaddr="buildbot@taler.net", # notify from pass to fail, and viceversa.