commit 878bf9502613b69ceeb83dbdf19faae6819c640d
parent 178216208c44281b5177267de53fea937ff1c5e6
Author: Devan Carpenter <devan@taler.net>
Date: Sun, 18 Jun 2023 16:42:10 -0400
buildbot: add support for container job configs
jobs can have configurations to specify paramaters, such as container
image, failure conditions, etc.
restructure container job loop to support generating jobs using the
built-in buildbot functions.
split logic into functions for improved clarity and composibility.
Diffstat:
| M | buildbot/master.cfg | | | 211 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- |
1 file changed, 175 insertions(+), 36 deletions(-)
diff --git a/buildbot/master.cfg b/buildbot/master.cfg
@@ -24,9 +24,13 @@
# @author ng0
# @author Christian Grothoff
# @author Devan Carpenter
+import ast
+import configparser
+import glob
import os
import pathlib
import re
+import subprocess
from buildbot.changes.pb import PBChangeSource
from buildbot.steps.source.git import Git
@@ -35,8 +39,10 @@ from buildbot.plugins import reporters
from buildbot.plugins import schedulers
from buildbot.plugins import steps
from buildbot.plugins import util
+from buildbot.process import buildstep, logobserver
from buildbot.reporters.generators.build import BuildStatusGenerator
from buildbot.worker import Worker
+from twisted.internet import defer
# This is a sample buildmaster config file. It must be
# installed as 'master.cfg' in your buildmaster's base
@@ -174,24 +180,47 @@ def update_deployment(factory):
# Convenience function that builds and runs a container.
-def container_run_step(stepName, factory, WORK_DIR, containerName,
+def container_add_step(HALT_ON_FAILURE,
+ WARN_ON_FAILURE,
+ CONTAINER_BUILD,
+ CONTAINER_NAME,
+ factory,
+ WORK_DIR,
+ stepName,
jobCmd="/workdir/ci/ci.sh",
containerFile="ci/Containerfile"):
- factory.addStep(steps.ShellSequence(
- name=stepName,
- commands=[
- util.ShellArg(command=["podman", "build", "-t", containerName,
- "-f", containerFile, "."],
- logname='build container', haltOnFailure=True),
- util.ShellArg(command=["podman", "run", "-ti", "--rm",
- "--volume", f"{WORK_DIR}:/workdir",
- "--workdir", "/workdir",
- containerName, jobCmd],
- logname='run inside container', haltOnFailure=True),
- ],
- haltOnFailure=True,
- workdir=WORK_DIR
- ))
+ print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}")
+ if not CONTAINER_BUILD:
+ return steps.ShellSequence(
+ name=stepName,
+ commands=[
+ util.ShellArg(command=["podman", "run", "-ti", "--rm",
+ "--volume", f"{WORK_DIR}:/workdir",
+ "--workdir", "/workdir",
+ CONTAINER_NAME, jobCmd],
+ logname='run inside container',
+ haltOnFailure=HALT_ON_FAILURE),
+ ],
+ haltOnFailure=HALT_ON_FAILURE,
+ workdir=WORK_DIR
+ )
+ else:
+ return steps.ShellSequence(
+ name=stepName,
+ commands=[
+ util.ShellArg(command=["podman", "build", "-t", CONTAINER_NAME,
+ "-f", containerFile, "."],
+ logname='build container', haltOnFailure=True),
+ util.ShellArg(command=["podman", "run", "-ti", "--rm",
+ "--volume", f"{WORK_DIR}:/workdir",
+ "--workdir", "/workdir",
+ CONTAINER_NAME, jobCmd],
+ logname='run inside container',
+ haltOnFailure=HALT_ON_FAILURE),
+ ],
+ haltOnFailure=HALT_ON_FAILURE,
+ workdir=WORK_DIR
+ )
##################################################################
######################## JOBS ####################################
@@ -1208,13 +1237,125 @@ WORKERS.append(Worker("container-worker", "container-pass"))
# "merchant", "deployment", "twister", "sync",
# "help", "taler-merchant-demos", "challenger"]
#
-container_repos = ["wallet-core"]
+
+
+# Container Job Generator Functions
+# Parse config file and save values in a dict
+def ingest_job_config(configPath, jobName):
+ configDict = {jobName: {}}
+ print(configDict)
+ ini.read_string(configPath)
+ for key in ini["build"]:
+ value = ini['build'][key]
+ configDict[jobName][key] = value
+ print(configDict)
+ configDict.update(configDict)
+ print(configDict)
+ return configDict
+
+
+# Search for configs, and ingest
+def handle_job_config(jobDirPath, jobName, repoName, configPath, configExist):
+ print(configPath)
+ if configExist == 0:
+ print(f"Ingesting Job Config: {configPath}")
+ configDict = ingest_job_config(configPath, jobName)
+ print(configDict)
+ return configDict
+ else:
+ print("No job config; Using default params")
+ # Set default job config parameters
+ configDict = {jobName: {"HALT_ON_FAILURE": True,
+ "WARN_ON_FAILURE": False,
+ "CONTAINER_BUILD": True,
+ "CONTAINER_NAME": repoName}}
+ return configDict
+
+
+class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep):
+
+ def __init__(self, reponame, **kwargs):
+ self.reponame = reponame
+ kwargs = self.setupShellMixin(kwargs)
+ super().__init__(**kwargs)
+ self.observer = logobserver.BufferLogObserver()
+ self.addLogObserver('stdio', self.observer)
+
+ def extract_stages(self, stdout):
+ stages = []
+ for line in stdout.split('\n'):
+ stage = str(line.strip())
+ if stage:
+ stages.append(stage)
+ return stages
+
+ @defer.inlineCallbacks
+ def run(self):
+ CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.reponame}"
+ CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/ci/jobs"
+ # run 'ls <project_root>/ci/jobs/' to get the list of stages
+ cmd = yield self.makeRemoteShellCommand()
+ yield self.runCommand(cmd)
+ jobDirs = []
+
+ # if the command passes extract the list of stages
+ result = cmd.results()
+ if result == util.SUCCESS:
+ jobDirs = self.extract_stages(self.observer.getStdout())
+ print(f"this is jobDirs list: {jobDirs}")
+ self.configDict = {}
+ print(f"Remote cmd stdout: {self.observer.getStdout()}")
+ print(f"cmd.results: {cmd.results()}")
+ for stage in jobDirs:
+ jobDirPath = f"{CI_JOBS_PATH}/{stage}"
+ observer = logobserver.BufferLogObserver()
+ self.addLogObserver('stdio', observer)
+ cmd1 = yield self.makeRemoteShellCommand(
+ command=["cat", f"{jobDirPath}/config.ini"])
+ yield self.runCommand(cmd1)
+ print(f"cmd1.results: {cmd1.results()}")
+ print(f"Second command stdout: {observer.getStdout()}")
+ print(f"Current stage: {stage}")
+ print(jobDirPath)
+ self.configDict.update(
+ handle_job_config(
+ jobDirPath, stage, self.reponame,
+ observer.getStdout(), cmd1.results()))
+ print(self.configDict)
+ # create a container step for each stage and
+ # add them to the build
+ convstr2bool = ast.literal_eval
+ self.build.addStepsAfterCurrentStep([
+ container_add_step(
+ convstr2bool(
+ str(self.configDict[stage]["HALT_ON_FAILURE"])),
+ convstr2bool(
+ str(self.configDict[stage]["WARN_ON_FAILURE"])),
+ convstr2bool(
+ str(self.configDict[stage]["CONTAINER_BUILD"])),
+ self.configDict[stage]["CONTAINER_NAME"],
+ container_factory,
+ CONTAINER_WORKDIR,
+ stage,
+ f"ci/jobs/{stage}/job.sh")
+ for stage in jobDirs
+ ])
+
+ return result
+
+
+container_repos = ["wallet-core", "merchant"]
for reponame in container_repos:
- ##
+
+ # Prepare to read job configs
+ ini = configparser.ConfigParser()
+ ini.optionxform = str
+
# Factory-wide variables
REPO_URL = "https://git.taler.net/" + reponame + ".git"
CONTAINER_WORKDIR = f"/home/container-worker/workspace/{reponame}"
+ CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/ci/jobs"
# Create a factory
container_factory = util.BuildFactory()
@@ -1237,37 +1378,35 @@ for reponame in container_repos:
container_factory.addStep(Git(
name="git",
repourl=REPO_URL,
+ branch="dev/devan-carpenter/container-ci",
mode='full',
method='fresh',
haltOnFailure=True,
))
- # Run container step with default commands
- CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/ci/jobs"
- if os.path.exists(CI_JOBS_PATH):
- for parentDir, dirNames, fileNames in os.walk(CI_JOBS_PATH):
- dirNames.sort()
- fileNames.sort()
- for filename in fileNames:
- if filename.endswith('.sh'):
- basedir = pathlib.PurePath(parentDir).name
- container_run_step(basedir,
- container_factory,
- CONTAINER_WORKDIR, reponame,
- f"ci/jobs/{basedir}/{filename}")
- else:
- print("No jobs found")
- else:
- print("Cannot find jobs directory")
+ container_factory.addStep(GenerateStagesCommand(
+ reponame,
+ name="Generate build stages",
+ command=["ls", CI_JOBS_PATH],
+ haltOnFailure=True))
BUILDERS.append(util.BuilderConfig(
name=f"{reponame}-builder",
workernames=["container-worker"],
factory=container_factory
))
+
+ # Only enable this scheduler for debugging!
+ # Will run builders with 1 minute of waiting inbetween builds
+ # SCHEDULERS.append(schedulers.Periodic(
+ # name=f"{reponame}-minutely",
+ # builderNames=[f"{reponame}-builder"],
+ # periodicBuildTimer=60
+ # ))
+
# Buildmaster is notified whenever deployment.git changes
SCHEDULERS.append(schedulers.SingleBranchScheduler(
- name="container-scheduler",
+ name=f"{reponame}-container-scheduler",
change_filter=util.ChangeFilter(
branch="dev/devan-carpenter/container-ci",
project_re=f"({reponame})"
