commit 3566669088b674cd6f8a5947941325c94c900cd5
parent c5d93947d4ae73ba74f2c216e7b5afe686e62701
Author: Christian Grothoff <christian@grothoff.org>
Date: Sun, 15 Dec 2019 16:31:11 +0100
documenting the master.cfg a bit better, minor refactoring, adding deployment checkout as first step to factories, and adding new job for updating the buildmaster if the deployment changes
Diffstat:
2 files changed, 99 insertions(+), 37 deletions(-)
diff --git a/bootstrap-buildmaster b/bootstrap-buildmaster
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Bootstrap the Taler setup for the buildmaster.
+
+set -eu
+
+BRANCH=master
+REPOS="deployment"
+
+cd $HOME
+
+for component in $REPOS; do
+ if ! test -d $HOME/$component; then
+ git clone git://git.taler.net/$component.git
+ fi
+done
+
+for component in $REPOS; do
+ echo "Checking out $component to $BRANCH"
+ git -C $HOME/$component checkout $BRANCH
+done
+
+if [[ ! -d worker ]]; then
+ buildbot-worker create-worker --umask=0o22 ~/worker localhost:9989 buildmaster-worker buildmaster-pass
+fi
diff --git a/buildbot/master.cfg b/buildbot/master.cfg
@@ -46,6 +46,11 @@ c["buildbotNetUsageData"] = "basic"
c["workers"] = [
##
+ # This worker restarts the buildmaster itself on
+ # changes to this file.
+ worker.Worker("buildmaster-worker", "buildmaster-pass"),
+
+ ##
# This worker makes the code coverage and publishes it
# under the "lcov" Website.
worker.Worker("lcov-worker", "lcov-pass"),
@@ -115,7 +120,7 @@ c["change_source"] = [ALLCS]
# Configure the Schedulers, which decide how to react to incoming
# changes.
-
+# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed:
def twister_web_page(change):
_change = change.asDict()
repo = _change.get("project")
@@ -127,10 +132,9 @@ def twister_web_page(change):
if re.search(r"web", file.get("name", "")) \
and "twister" == repo:
return True
-
return False
-
+# Jobs we run at 6am every day: coverage analysis and auditor.
NIGHTLY_SCHEDULER = schedulers.Nightly(
name="nightly-scheduler",
builderNames=["lcov-builder",
@@ -141,15 +145,17 @@ NIGHTLY_SCHEDULER = schedulers.Nightly(
minute=0
)
+# Docs run if master or stable branch of 'docs' (or deployment) changed.
DOC_SCHEDULER = schedulers.SingleBranchScheduler(
name="periodic-doc-scheduler",
builderNames=["doc-builder"],
change_filter=util.ChangeFilter(
- branch_re="(master|stable)", project_re="(docs)"
+ branch_re="(master|stable)", project_re="(docs|deployment)"
),
treeStableTimer=None
)
+# Sites are re-build whenever deployment, www or twister changes.
SITES_SCHEDULER = schedulers.SingleBranchScheduler(
name="sites-scheduler",
builderNames=["sites-builder"],
@@ -157,48 +163,43 @@ SITES_SCHEDULER = schedulers.SingleBranchScheduler(
branch_re="(master|stable)",
# Given that filter_fn is used, the line below could be
# removed (?)
- project_re="(www|twister)",
+ project_re="(www|twister|deployment)",
filter_fn=twister_web_page
),
treeStableTimer=None
)
+# Wallet is re-build whenever wallet-core or deployment changes
WALLET_SCHEDULER = schedulers.SingleBranchScheduler(
name="wallet-scheduler",
change_filter=util.ChangeFilter(
branch="master",
- project_re="(wallet|deployment)"
+ project_re="(wallet-core|deployment)"
),
treeStableTimer=None,
builderNames=["wallet-builder"]
)
-TALER_TEST_HEALTHCHECK = schedulers.SingleBranchScheduler(
- name="taler-test-healthcheck-scheduler",
+# Healthchecks are done on wallet or deployment changes to master
+TALER_HEALTHCHECK_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="taler-healthcheck-scheduler",
change_filter=util.ChangeFilter(
branch="master",
project_re="(wallet|deployment)"
),
treeStableTimer=None,
- builderNames=["taler-test-healthcheck-builder"]
-)
-
-TALER_DEMO_HEALTHCHECK = schedulers.SingleBranchScheduler(
- name="taler-demo-healthcheck-scheduler",
- change_filter=util.ChangeFilter(
- branch="master",
- project_re="(wallet|deployment)"
- ),
- treeStableTimer=None,
- builderNames=["taler-demo-healthcheck-builder"]
+ builderNames=["taler-test-healthcheck-builder",
+ "taler-demo-healthcheck-builder"]
)
+# We check demo once per hour.
DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic(
name="demo-services-checker-scheduler",
periodicBuildTimer=60 * 60, # 1 hour
builderNames=["demo-services-checker-builder"]
)
+# We top-up the tipping reserve every 10 days.
TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic(
name="tip-reserve-topper-scheduler",
periodicBuildTimer=60 * 60 * 24 * 10, # 10 days
@@ -206,11 +207,12 @@ TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic(
"tip-reserve-topper-builder-test"]
)
+# Scheduler that triggers if anything changes
ALL_SCHEDULER = schedulers.SingleBranchScheduler(
name="all-scheduler",
change_filter=util.ChangeFilter(
branch_re="(master|stable)",
- project_re="(backoffice|wallet|bank|exchange|"
+ project_re="(backoffice|wallet-core|bank|exchange|"
"merchant|deployment|donations|twister|"
"blog|help|survey|landing)"
),
@@ -220,6 +222,7 @@ ALL_SCHEDULER = schedulers.SingleBranchScheduler(
# Scheduler monitoring the help.git repo; a forgotten repo we
# use to test BB.
+# NOTE: scheduler is NOT active! (commented out below)
DEBUG_SCHEDULER = schedulers.SingleBranchScheduler(
name="debug-scheduler",
change_filter=util.ChangeFilter(branch="master", project="help"),
@@ -228,6 +231,7 @@ DEBUG_SCHEDULER = schedulers.SingleBranchScheduler(
)
# Consider adding other Python parts, like the various frontends.
+# NOTE: scheduler is NOT active! (commented out below)
LINT_SCHEDULER = schedulers.SingleBranchScheduler(
name="lint-scheduler",
change_filter=util.ChangeFilter(
@@ -237,8 +241,7 @@ LINT_SCHEDULER = schedulers.SingleBranchScheduler(
builderNames=["lint-builder"]
)
-# Provide "force" button in the web UI. To be removed in the
-# future ?
+# Provide "force" button in the web UI.
FORCE_SCHEDULER = schedulers.ForceScheduler(
name="force-scheduler",
builderNames=[
@@ -265,8 +268,7 @@ c["schedulers"] = [
ALL_SCHEDULER,
FORCE_SCHEDULER,
WALLET_SCHEDULER,
- TALER_DEMO_HEALTHCHECK,
- TALER_TEST_HEALTHCHECK,
+ TALER_HEALTHCHECK_SCHEDULER,
##
# Rarely/never used, excluding.
@@ -282,7 +284,10 @@ c["schedulers"] = [
# one worker.
-def git_step(repo):
+# Convenience function that checks out a Git repository.
+# First argument is the URL of the Git to clone, second
+# the desired branch. Default is 'master'.
+def git_step(repo,target_branch="master"):
return Git(
repourl=repo,
mode="full",
@@ -290,10 +295,32 @@ def git_step(repo):
logEnviron=False,
alwaysUseLatest=True,
haltOnFailure=True,
- branch="master"
+ branch=target_branch
+ )
+
+# Convenience function that checks out the deployment.
+def update_deployment(factory):
+ return factory.addStep(git_step("git://git.taler.net/deployment.git"))
+
+# Create a FACTORY with a deployment.git checkout as the first step.
+def create_factory_with_deployment():
+ f = util.BuildFactory()
+ update_deployment (f);
+ return f
+
+# Factory for a job to update the buildmaster itself.
+BUILDMASTER_FACTORY = create_factory_with_deployment()
+BUILDMASTER_FACTORY.addStep(
+ ShellCommand(
+ name="restart buildmaster",
+ description="trigger buildmaster restart with new configuration",
+ descriptionDone="Buildmaster updated",
+ command=["buildbot", "reconfig"],
+ workdir="master/"
)
+)
-TALER_DEMO_HEALTHCHECK_FACTORY = util.BuildFactory()
+TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment()
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
ShellCommand(
@@ -323,7 +350,7 @@ TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
)
)
-TALER_TEST_HEALTHCHECK_FACTORY = util.BuildFactory()
+TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment()
TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
TALER_TEST_HEALTHCHECK_FACTORY.addStep(
ShellCommand(
@@ -354,7 +381,7 @@ TALER_TEST_HEALTHCHECK_FACTORY.addStep(
)
-WALLET_FACTORY = util.BuildFactory()
+WALLET_FACTORY = create_factory_with_deployment()
WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
WALLET_FACTORY.addStep(
ShellCommand(
@@ -437,7 +464,7 @@ LINT_FACTORY.addStep(
)
)
-LCOV_FACTORY = util.BuildFactory()
+LCOV_FACTORY = create_factory_with_deployment()
LCOV_FACTORY.addStep(
ShellCommand(
name="build",
@@ -461,7 +488,7 @@ LCOV_FACTORY.addStep(
)
# FIXME: 'demo' reports generator missing.
-AUDITOR_FACTORY = util.BuildFactory()
+AUDITOR_FACTORY = create_factory_with_deployment()
AUDITOR_FACTORY.addStep(
ShellCommand(
name="Auditor reports generator",
@@ -472,7 +499,7 @@ AUDITOR_FACTORY.addStep(
)
)
-TIP_RESERVE_TOPPER_FACTORY = util.BuildFactory()
+TIP_RESERVE_TOPPER_FACTORY = create_factory_with_deployment()
TIP_RESERVE_TOPPER_FACTORY.addStep(
ShellCommand(
name="tip reserve topper",
@@ -483,7 +510,7 @@ TIP_RESERVE_TOPPER_FACTORY.addStep(
)
)
-BUILD_FACTORY = util.BuildFactory()
+BUILD_FACTORY = create_factory_with_deployment()
BUILD_FACTORY.addStep(
ShellCommand(
name="build",
@@ -554,7 +581,7 @@ BUILD_FACTORY.addStep(
)
)
-SELENIUM_FACTORY = util.BuildFactory()
+SELENIUM_FACTORY = create_factory_with_deployment()
SELENIUM_FACTORY.addStep(
ShellCommand(
name="selenium",
@@ -565,7 +592,7 @@ SELENIUM_FACTORY.addStep(
)
)
-DOC_FACTORY = util.BuildFactory()
+DOC_FACTORY = create_factory_with_deployment()
DOC_FACTORY.addStep(
ShellCommand(
name="build docs",
@@ -577,7 +604,7 @@ DOC_FACTORY.addStep(
)
)
-SITES_FACTORY = util.BuildFactory()
+SITES_FACTORY = create_factory_with_deployment()
SITES_FACTORY.addStep(
ShellCommand(
name="build Web sites",
@@ -589,7 +616,7 @@ SITES_FACTORY.addStep(
)
)
-DEMO_SERVICES_CHECKER_FACTORY = util.BuildFactory()
+DEMO_SERVICES_CHECKER_FACTORY = create_factory_with_deployment()
DEMO_SERVICES_CHECKER_FACTORY.addStep(
ShellCommand(
name="demo services checker",
@@ -609,6 +636,14 @@ DEBUG_BUILDER = util.BuilderConfig(
factory=DEBUG_FACTORY
)
+# Builder to update the buildmaster itself.
+BUILDMASTER_BUILDER = util.BuilderConfig(
+ name="buildmaster-builder",
+ workernames=["buildmaster-worker"],
+ factory=BUILDMASTER_FACTORY
+)
+
+# This builder is NOT ACTIVE!
LINT_BUILDER = util.BuilderConfig(
name="lint-builder",
workernames=["lint-worker"],
@@ -688,6 +723,7 @@ TALER_DEMO_HEALTHCHECK_BUILDER = util.BuilderConfig(
)
c["builders"] = [
+ BUILDMASTER_BUILDER,
LCOV_BUILDER,
AUDITOR_BUILDER_TEST,
AUDITOR_BUILDER_DEMO,
@@ -755,6 +791,7 @@ c["buildbotURL"] = "https://buildbot.taler.net/"
from taler_bb_userpass_db import USER_PASSWORD_DB
BUILDER_LIST = [
+ "buildmaster-builder",
"doc-builder",
"lcov-builder",
"auditor-builder",
