commit fb1228ac018ef8295f403acc327ce58dcd535fbf
parent 17d783c133089b6212a72448daf4d1e0989d8106
Author: Christian Grothoff <christian@grothoff.org>
Date: Tue, 10 Nov 2026 01:53:46 +0100
buildbot: run nightly-only CI jobs from a nightly scheduler
Stages are discovered by listing contrib/ci/jobs, which cannot express "not on
every push", so a job now declares RUN_ON in its config.ini and the stage skips
itself in the builds it is not for. taler-android's three-hour emulator suite is
the first to need it; onlyIfChanged keeps a day with no pushes quiet.
Assisted by LLM
Diffstat:
| M | buildbot/master.cfg | | | 73 | ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- |
1 file changed, 70 insertions(+), 3 deletions(-)
diff --git a/buildbot/master.cfg b/buildbot/master.cfg
@@ -216,7 +216,8 @@ def container_add_step(HALT_ON_FAILURE,
stepName,
CONTAINER_ARCH="amd64",
jobCmd="/workdir/contrib/ci/ci.sh",
- containerFile="contrib/ci/Containerfile"):
+ containerFile="contrib/ci/Containerfile",
+ RUN_ON="push"):
print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}")
# Locally built images belong to this repository, job and architecture.
@@ -271,6 +272,29 @@ def container_add_step(HALT_ON_FAILURE,
"-f", containerFile, "."],
logname='build container', haltOnFailure=True)
+ # Stages come from `ls contrib/ci/jobs`, which cannot express "not in this
+ # kind of build". So each job says which schedulers it is for and the
+ # stage skips itself in the others. Without this a nightly would re-run
+ # the deployment jobs and publish a second set of nightly APKs, and a push
+ # would wait for a job meant to run once a day.
+ #
+ # `scheduler` is set by buildbot on every build a scheduler triggers, so
+ # the two names below are the two automatic cases and everything else is
+ # somebody asking for a build by hand.
+ def runs_in_this_build(step):
+ scheduler = step.getProperty("scheduler", "") or ""
+ if scheduler == f"{repoName}-nightly":
+ return RUN_ON in ("nightly", "both")
+ if scheduler == f"{repoName}-container-scheduler":
+ return RUN_ON in ("push", "both")
+ # Anything else -- a forced build, or a repository wired up by hand
+ # rather than through the container loop -- runs every stage. It was
+ # asked for explicitly, and a nightly-only job that could not be
+ # forced would be a job nobody could try out before tomorrow.
+ return True
+
+ skipped = lambda results, s: results == SKIPPED
+
if not CONTAINER_BUILD:
return steps.ShellSequence(
name=stepName,
@@ -278,6 +302,8 @@ def container_add_step(HALT_ON_FAILURE,
haltOnFailure=HALT_ON_FAILURE,
flunkOnFailure=True,
warnOnWarnings=True,
+ doStepIf=runs_in_this_build,
+ hideStepIf=skipped,
workdir=WORK_DIR
)
else:
@@ -287,6 +313,8 @@ def container_add_step(HALT_ON_FAILURE,
haltOnFailure=HALT_ON_FAILURE,
flunkOnFailure=True,
warnOnWarnings=True,
+ doStepIf=runs_in_this_build,
+ hideStepIf=skipped,
workdir=WORK_DIR
)
@@ -942,7 +970,14 @@ def default_job_config(jobName, repoName):
"WARN_ON_FAILURE": False,
"CONTAINER_BUILD": True,
"CONTAINER_NAME": repoName,
- "CONTAINER_ARCH": "amd64"}}
+ "CONTAINER_ARCH": "amd64",
+ # Which schedulers a job is for: "push", "nightly" or
+ # "both". The default keeps every existing job exactly
+ # where it was -- stages are discovered by listing
+ # contrib/ci/jobs, so a job that said nothing used to
+ # run in every build of its repository, and a repository
+ # without a nightly scheduler still has only one kind.
+ "RUN_ON": "push"}}
# Parse a job config and apply it as overrides to the standard configuration.
@@ -1036,7 +1071,8 @@ class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
self.REPO_NAME,
stage,
self.configDict[stage]["CONTAINER_ARCH"],
- f"contrib/ci/jobs/{stage}/job.sh")
+ f"contrib/ci/jobs/{stage}/job.sh",
+ RUN_ON=self.configDict[stage]["RUN_ON"])
for stage in jobDirs
])
@@ -1094,6 +1130,12 @@ container_repos = ["git.gnunet.org/gnunet",
"git.taler.net/taldir",
"git.taler.net/taler-typescript-core",]
+# Repositories with at least one `RUN_ON = nightly` job in contrib/ci/jobs.
+# Listed rather than inferred because the stage list is only known once a
+# build is running, while a scheduler has to exist before that: a nightly for
+# a repository without such a job would be a build whose every stage skips.
+nightly_container_repos = ["taler-android"]
+
for repo in container_repos:
# Factory-wide variables
@@ -1184,6 +1226,31 @@ for repo in container_repos:
builderNames=[f"{REPO_NAME}-builder"]
))
+ # Jobs whose config.ini says `RUN_ON = nightly` run from here instead of
+ # on every push: the android integration suite drives a real emulator and
+ # takes about three hours, which is not something to put in front of the
+ # next push to a worker shared with every other repository.
+ #
+ # onlyIfChanged keeps a quiet day quiet -- no change to this repository
+ # since the last build means no build tonight. Note that changes arrive
+ # here from the git hook (PBChangeSource), not from a poller, so a
+ # repository whose hook is silent gets no nightly rather than a redundant
+ # one.
+ #
+ # The builder is the same one the push scheduler uses; what differs is the
+ # `scheduler` property, which is how container_add_step decides whether a
+ # given stage belongs in this build.
+ if REPO_NAME in nightly_container_repos:
+ SCHEDULERS.append(schedulers.Nightly(
+ name=f"{REPO_NAME}-nightly",
+ change_filter=util.ChangeFilter(
+ branch="master",
+ project_re=f"^({REPO_NAME})$"
+ ),
+ builderNames=[f"{REPO_NAME}-builder"],
+ onlyIfChanged=True,
+ hour=2, minute=30))
+
SERVICES.append(reporters.MailNotifier(
fromaddr="buildbot@taler.net",
# notify from pass to fail, and viceversa.