commit aec67d1d9af4dc155d759571f5ec69138fd1dcc4
parent dfb13f785c69a61d0e38996eaf7972a81c64e03f
Author: buckE <buckE@disroot.org>
Date: Wed, 17 Jun 2020 05:31:41 +0000
moved create_factory_with_deployment function above declarations
Diffstat:
2 files changed, 1162 insertions(+), 6 deletions(-)
diff --git a/buildbot/master.cfg b/buildbot/master.cfg
@@ -37,14 +37,22 @@ c = BuildmasterConfig = {}
# Silence warning and allow very basic phoning home.
c["buildbotNetUsageData"] = "basic"
+
+# Create a FACTORY with a deployment.git checkout as the first step.
+def create_factory_with_deployment():
+ f = util.BuildFactory()
+ update_deployment (f);
+ return f
+
###########################################################################################
# ------------- NEW FORMATTING STARTS HERE FOR NEW AND MODIFIED PROJECTS -----------------
+# !!! This matters. Mis-ordering leads to failure. !!!
# Best Practices: Group by worker as:
# - comment for worker location (ex: "Find worker at: xyz shell account non taler.net")
# - SCHEDULER
-# - BUILDER function
# - FACTORY function
+# - BUILDER function
#
# NOTE: WORKER must be added to list: c["workers"] = [
# NOTE: BUILDER must be added to list: c["builders"] = [
@@ -443,11 +451,6 @@ def update_deployment(factory):
workdir="../.."
))
-# Create a FACTORY with a deployment.git checkout as the first step.
-def create_factory_with_deployment():
- f = util.BuildFactory()
- update_deployment (f);
- return f
# Factory to update the buildmaster itself.
BUILDMASTER_FACTORY = create_factory_with_deployment()
diff --git a/buildbot/master.cfg.BACK-movingFunctionsAround b/buildbot/master.cfg.BACK-movingFunctionsAround
@@ -0,0 +1,1153 @@
+##
+# This file is part of TALER
+# (C) 2016-2020 Taler Systems SA
+#
+# TALER is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Affero General Public
+# License as published by the Free Software Foundation; either
+# version 3, or (at your option) any later version.
+#
+# TALER is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty
+# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public
+# License along with TALER; see the file COPYING. If not,
+# see <http://www.gnu.org/licenses/>
+#
+# @author Florian Dold
+# @author Marcello Stanisci
+# @author ng0
+# @author Christian Grothoff
+import re
+from getpass import getuser
+from buildbot.steps.source.git import Git
+from buildbot.steps.shell import ShellCommand
+from buildbot.plugins import (reporters, worker,
+ changes, schedulers, util)
+
+# This is a sample buildmaster config file. It must be
+# installed as 'master.cfg' in your buildmaster's base
+# directory. This is the dictionary that the buildmaster
+# pays attention to. We also use a shorter alias to save
+# typing.
+c = BuildmasterConfig = {}
+
+# Silence warning and allow very basic phoning home.
+c["buildbotNetUsageData"] = "basic"
+
+###########################################################################################
+# ------------- NEW FORMATTING STARTS HERE FOR NEW AND MODIFIED PROJECTS -----------------
+
+# !!! This matters. Mis-ordering leads to failure. !!!
+# Best Practices: Group by worker as:
+# - comment for worker location (ex: "Find worker at: xyz shell account non taler.net")
+# - SCHEDULER
+# - FACTORY function
+# - BUILDER function
+#
+# NOTE: WORKER must be added to list: c["workers"] = [
+# NOTE: BUILDER must be added to list: c["builders"] = [
+# NOTE: BUILDER must be added to this list too: BUILDER_LIST = [
+###########################################################################################
+
+###########################
+### --- buildslavetest ---
+
+# buildslavetest WORKER: buildslavetest@taler.net
+
+# buildslavetest SCHEDULER
+# Run at 11am every day.
+TEST_SCHEDULER = schedulers.Nightly(
+ name="test-scheduler",
+ builderNames=["buildslavetest-builder"],
+ branch="master",
+ hour=11,
+ minute=00
+)
+
+# buildslavetest FACTORY
+BUILDSLAVETEST_FACTORY = create_factory_with_deployment()
+BUILDSLAVETEST_FACTORY.addStep(
+ ShellCommand(
+ name="buildslavetest script (for testing purposes)",
+ description="Build Slave Test",
+ descriptionDone="buildslavetest: Done",
+ command=["./buildslavetest.sh"],
+ workdir="/home/buildslavetest/"
+ )
+)
+
+# buildslavetest BUILDER
+BUILDSLAVETEST_BUILDER = util.BuilderConfig(
+ name="buildslavetest-builder",
+ workernames=["buildslavetest-worker"],
+ factory=BUILDSLAVETEST_FACTORY
+)
+
+### /end ---buildslavetest
+
+#######################
+### --- linkchecker ---
+
+# linkchecker WORKER Location: linkchecker@taler.net
+
+# linkchecker SCHEDULER
+# Run at 7:30am every day.
+LINKCHECKER_SCHEDULER = schedulers.Nightly(
+ name="linkchecker-scheduler",
+ builderNames="linkchecker-builder",
+ branch="master",
+ hour=7,
+ minute=30
+)
+
+# linkchecker BUILDER
+# Builder for linkchecker
+LINKCHECKER_BUILDER = util.BuilderConfig(
+ name="linkchecker-builder",
+ workernames="linkchecker-worker",
+ factory=LINKCHECKER_FACTORY
+)
+
+DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig(
+ name="demo-services-checker-builder",
+ workernames="demo-worker",
+ factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY
+)
+
+# linkchecker FACTORY
+LINKCHECKER_FACTORY = create_factory_with_deployment()
+LINKCHECKER_FACTORY.addStep(
+ ShellCommand(
+ name="linkchecker",
+ description="Check taler.net website for broken links && Notify",
+ descriptionDone="linkchecker links checked",
+ command=["./linkchecker.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True
+ )
+)
+
+### /end --- linkchecker ---
+
+# / ##########################################################################################
+
+
+####### WORKERS
+
+# The 'workers' list defines the set of recognized workers.
+# Each element is a Worker object, specifying a unique worker
+# name and password. The same worker name and password must
+# be configured on the worker.
+c["workers"] = [
+
+ ##
+ # This worker restarts the buildmaster itself on
+ # changes to this file.
+ worker.Worker("buildmaster-worker", "buildmaster-pass"),
+
+ ##
+ # This worker makes the code coverage and publishes it
+ # under the "lcov" Website.
+ worker.Worker("lcov-worker", "lcov-pass"),
+
+ ##
+ # This worker builds everything and runs our 'make check'
+ # test suite against 'everything'.
+ worker.Worker("checker-worker", "checker-pass"),
+
+ ##
+ # This worker builds manuals / API docs / tutorials.
+ #
+ worker.Worker("doc-worker", "doc-pass"),
+
+ ##
+ # This worker builds Websites: www and stage.
+ #
+ worker.Worker("sites-worker", "sites-pass"),
+
+ ##
+ # This worker builds Taler for the 'test' deployment.
+ worker.Worker("test-worker", "test-pass"),
+
+ ##
+ # Tip reserve toppers.
+ worker.Worker("test-topper-worker", "test-topper-pass"),
+ worker.Worker("demo-topper-worker", "demo-topper-pass"),
+
+ ##
+ # This worker compiles the auditor reports for the "green"
+ # demo deployment.
+ worker.Worker("test-auditor-worker", "test-auditor-pass"),
+ worker.Worker("demo-auditor-worker", "demo-auditor-pass"),
+
+ ##
+ # This worker checks that all the services run under the
+ # 'demo' deployment are up&running.
+ worker.Worker("demo-worker", "demo-pass"),
+
+ ##
+ # This worker builds wallet-core.
+ worker.Worker("wallet-worker", "wallet-pass"),
+
+ ##
+ # health checks performed by wallet-cli for demo
+ worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass"),
+
+ ##
+ # health checks performed by wallet-cli for test
+ worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass"),
+
+ ##
+ # testing buildbot using the "buildslavetest" user (for no specific reason except it exists)
+ # Home: `buidlslavetest` @ taler.net
+ worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo"),
+
+ ##
+ # linkchecker worker checks for dead links in the Website
+ # Home: linkchecker@taler.net
+ worker.Worker("linkchecker-worker", "linkchecker-worker-pass")
+]
+
+# 'protocols' contains information about protocols which master
+# will use for communicating with workers. You must define at
+# least 'port' option that workers could connect to your master
+# with this protocol. 'port' must match the value configured into
+# the workers (with their --master option)
+c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}}
+
+####### CHANGESOURCES
+
+# the 'change_source' setting tells the buildmaster how it should
+# find out about source code changes.
+
+# NOTE: BB is bound to localhost
+ALLCS = changes.PBChangeSource(user="allcs", passwd="allcs")
+
+c["change_source"] = [ALLCS]
+
+####### SCHEDULERS
+
+# Configure the Schedulers, which decide how to react to incoming
+# changes.
+
+# Run at 6am every day: coverage analysis and auditor.
+NIGHTLY_SCHEDULER = schedulers.Nightly(
+ name="nightly-scheduler",
+ builderNames=["lcov-builder",
+ "auditor-builder-demo",
+ "auditor-builder-test"],
+ branch="master",
+ hour=6,
+ minute=0
+)
+
+# Docs run if master or stable branch of 'docs' (or deployment) changed.
+DOC_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="periodic-doc-scheduler",
+ builderNames=["doc-builder"],
+ change_filter=util.ChangeFilter(
+ branch_re="(master|stable)", project_re="(docs|deployment)"
+ ),
+ treeStableTimer=None
+)
+
+
+# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed:
+def twister_web_page(change):
+ _change = change.asDict()
+ repo = _change.get("project")
+ if repo in ["www", "buywith", "deployment"]:
+ return True
+
+ files = _change.get("files")
+ for file in files:
+ if re.search(r"web", file.get("name", "")) \
+ and "twister" == repo:
+ return True
+ return False
+
+# Sites are re-build whenever deployment, www buywith, or twister changes.
+SITES_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="sites-scheduler",
+ builderNames=["sites-builder"],
+ change_filter=util.ChangeFilter(
+ branch_re="(master|stable)",
+ filter_fn=twister_web_page
+ ),
+ treeStableTimer=None
+)
+
+# Buildmaster is notified whenever deployment changes
+BUILDMASTER_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="buildmaster-scheduler",
+ change_filter=util.ChangeFilter(
+ branch="master",
+ project_re="(deployment)"
+ ),
+ treeStableTimer=None,
+ builderNames=["buildmaster-builder"]
+)
+
+# Wallet is re-build whenever wallet-core or deployment changes
+WALLET_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="wallet-scheduler",
+ change_filter=util.ChangeFilter(
+ branch="master",
+ project_re="(wallet-core|deployment)"
+ ),
+ treeStableTimer=None,
+ builderNames=["wallet-builder"]
+)
+
+# Healthchecks are done on wallet or deployment changes to master
+TALER_HEALTHCHECK_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="taler-healthcheck-scheduler",
+ change_filter=util.ChangeFilter(
+ branch="master",
+ project_re="(wallet|deployment)"
+ ),
+ treeStableTimer=None,
+ builderNames=["taler-test-healthcheck-builder",
+ "taler-demo-healthcheck-builder"]
+)
+
+# We check demo once per hour.
+DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic(
+ name="demo-services-checker-scheduler",
+ periodicBuildTimer=60 * 60, # 1 hour
+ builderNames=["demo-services-checker-builder"]
+)
+
+# We top-up the tipping reserve every 10 days.
+TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic(
+ name="tip-reserve-topper-scheduler",
+ periodicBuildTimer=60 * 60 * 24 * 10, # 10 days
+ builderNames=["tip-reserve-topper-builder-demo",
+ "tip-reserve-topper-builder-test"]
+)
+
+# Scheduler that triggers if anything changes
+ALL_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="all-scheduler",
+ change_filter=util.ChangeFilter(
+ branch_re="(master|stable)",
+ project_re="(backoffice|wallet-core|bank|exchange|"
+ "merchant|deployment|donations|twister|sync|"
+ "blog|help|survey|landing)"
+ ),
+ treeStableTimer=None,
+ builderNames=["test-builder", "checker-builder"]
+)
+
+# Consider adding other Python parts, like the various frontends.
+# NOTE: scheduler is NOT active! (commented out below)
+LINT_SCHEDULER = schedulers.SingleBranchScheduler(
+ name="lint-scheduler",
+ change_filter=util.ChangeFilter(
+ branch="master", project_re="(bank|donations|survey|blog)"
+ ),
+ treeStableTimer=None,
+ builderNames=["lint-builder"]
+)
+
+# Provide "force" button in the web UI.
+FORCE_SCHEDULER = schedulers.ForceScheduler(
+ name="force-scheduler",
+ builderNames=[
+ "lcov-builder",
+ "checker-builder",
+ "auditor-builder-test",
+ "auditor-builder-demo",
+ "test-builder",
+ "doc-builder",
+ "linkchecker-builder",
+ "buildslavetest-builder",
+ "sites-builder",
+ "wallet-builder",
+ "taler-test-healthcheck-builder",
+ "taler-demo-healthcheck-builder",
+ "tip-reserve-topper-builder-test",
+ "tip-reserve-topper-builder-demo"
+ ]
+)
+
+c["schedulers"] = [
+ TEST_SCHEDULER,
+ NIGHTLY_SCHEDULER,
+ TIP_RESERVE_TOPPER_SCHEDULER,
+ DEMO_SERVICES_CHECKER_SCHEDULER,
+ DOC_SCHEDULER,
+ BUILDMASTER_SCHEDULER,
+ SITES_SCHEDULER,
+ ALL_SCHEDULER,
+ FORCE_SCHEDULER,
+ WALLET_SCHEDULER,
+ TALER_HEALTHCHECK_SCHEDULER,
+
+ ##
+ # Rarely/never used, excluding.
+ # DEBUG_SCHEDULER,
+ # LINT_SCHEDULER
+]
+
+####### BUILDERS
+
+# The 'builders' list defines the Builders, which tell Buildbot
+# how to perform a build: what steps, and which workers can execute
+# them. Note that any particular build will only take place on
+# one worker.
+
+
+# Convenience function that checks out a Git repository.
+# First argument is the URL of the Git to clone, second
+# the desired branch. Default is 'master'.
+def git_step(repo,target_branch="master"):
+ return Git(
+ repourl=repo,
+ mode="full",
+ method="fresh",
+ logEnviron=False,
+ alwaysUseLatest=True,
+ haltOnFailure=True,
+ branch=target_branch
+ )
+
+
+# Convenience function that runs 'make check' in a
+# directory of the code inside of a netjail.
+def jailed_check(package,srcdir):
+ return ShellCommand(
+ name="Test of " + srcdir + " in " + package,
+ description="Testing " + srcdir + " in " + package,
+ descriptionDone="Pass",
+ command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"],
+ workdir="../../sources/" + package + "/src/" + srcdir
+ )
+
+# Convenience function that checks out the deployment.
+def update_deployment(factory):
+ factory.addStep(ShellCommand(
+ name="clean old deployment",
+ description="cleaning previous deployment repository",
+ descriptionDone="Deployment removed",
+ command=["rm", "-rf", "deployment"],
+ workdir="../.."
+ ))
+ factory.addStep(ShellCommand(
+ name="update deployment",
+ description="fetching latest deployment repository",
+ descriptionDone="Deployment updated",
+ command=["git", "clone", "git://git.taler.net/deployment"],
+ workdir="../.."
+ ))
+
+# Create a FACTORY with a deployment.git checkout as the first step.
+def create_factory_with_deployment():
+ f = util.BuildFactory()
+ update_deployment (f);
+ return f
+
+# Factory to update the buildmaster itself.
+BUILDMASTER_FACTORY = create_factory_with_deployment()
+BUILDMASTER_FACTORY.addStep(
+ ShellCommand(
+ name="restart buildmaster",
+ description="trigger buildmaster restart with new configuration",
+ descriptionDone="Buildmaster updated",
+ command=["buildbot", "reconfig"],
+ workdir="../../master/"
+ )
+)
+
+TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment()
+TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
+TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running yarn install",
+ descriptionDone="Correctly installed",
+ command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"],
+ workdir="build/"
+ )
+)
+TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
+ ShellCommand(
+ name="test-withdraw",
+ description="Running wallet withdraw tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-w", "TESTKUDOS:10"],
+ workdir="build/",
+ )
+)
+TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
+ ShellCommand(
+ name="test-spend",
+ description="Running wallet spend tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-s", "TESTKUDOS:4"],
+ workdir="build/",
+ )
+)
+
+TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment()
+TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
+TALER_TEST_HEALTHCHECK_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running yarn install",
+ descriptionDone="Correctly installed",
+ command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"],
+ workdir="build/"
+ )
+)
+TALER_TEST_HEALTHCHECK_FACTORY.addStep(
+ ShellCommand(
+ name="test-withdraw",
+ description="Running wallet withdraw tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"],
+ workdir="build/",
+ )
+)
+TALER_TEST_HEALTHCHECK_FACTORY.addStep(
+ ShellCommand(
+ name="test-spend",
+ description="Running wallet spend tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"],
+ workdir="build/",
+ )
+)
+
+
+WALLET_FACTORY = create_factory_with_deployment()
+WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
+WALLET_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running yarn install",
+ descriptionDone="Correctly installed",
+ command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"],
+ workdir="build/"
+ )
+)
+WALLET_FACTORY.addStep(
+ ShellCommand(
+ name="test",
+ description="Running wallet tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose"],
+ workdir="build/",
+ )
+)
+WALLET_FACTORY.addStep(
+ ShellCommand(
+ name="test-withdraw",
+ description="Running wallet withdraw tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-w", "TESTKUDOS:10"],
+ workdir="build/",
+ )
+)
+WALLET_FACTORY.addStep(
+ ShellCommand(
+ name="test-spend",
+ description="Running wallet spend tests",
+ descriptionDone="Test correctly run",
+ command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-s", "TESTKUDOS:4"],
+ workdir="build/",
+ )
+)
+# WALLET_FACTORY.addStep(
+# ShellCommand(
+# name="lint",
+# description="Linting the wallet",
+# descriptionDone="Linting done",
+# command=["make", "lint"],
+# workdir="build/"
+# )
+# )
+
+DEBUG_FACTORY = util.BuildFactory()
+DEBUG_FACTORY.addStep(
+ ShellCommand(
+ name="echo debug",
+ description="just echoing a word",
+ descriptionDone="builder responded",
+ command=["echo", "I'm here!"]
+ )
+)
+
+
+def lint_dispatcher(project):
+ return "./lint_%s.sh" % project
+
+
+LINT_FACTORY = util.BuildFactory()
+LINT_FACTORY.addStep(
+ ShellCommand(
+ name="Python linter",
+ description="linting Python",
+ descriptionDone="linting done",
+ command=util.Transform(lint_dispatcher, util.Property("project")),
+ workdir="../../deployment/taler-build"
+ )
+)
+
+
+INTEGRATIONTEST_FACTORY = create_factory_with_deployment()
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="remove old binaries",
+ description="removing binaries from previous runs",
+ descriptionDone="Setup clean",
+ command=["rm", "-rf", "local"],
+ workdir="../.."
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="fetch gnunet.org sources",
+ description="fetching latest deployment repositories from git.gnunet.org",
+ descriptionDone="GNUnet code base updated",
+ command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"],
+ workdir="../../deployment/buildbot"
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="fetch taler.net sources",
+ description="fetching latest deployment repositories from git.taler.net",
+ descriptionDone="Taler code base updated",
+ command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "anastasis", "bank", "twister"],
+ workdir="../../deployment/buildbot"
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running yarn install of wallet",
+ descriptionDone="Correctly installed",
+ command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"],
+ workdir="../../sources/wallet-core"
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running local bootstrap of wallet",
+ descriptionDone="wallet bootstrap done",
+ command=["bootstrap"],
+ workdir="../../sources/wallet-core"
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running local configure of wallet",
+ descriptionDone="wallet configure done",
+ command=["configure", "--prefix=$HOME/local/"],
+ workdir="../../sources/wallet-core"
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running local make of wallet",
+ descriptionDone="wallet make done",
+ command=["make"],
+ workdir="../../sources/wallet-core"
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="build",
+ description="Building other Taler components",
+ descriptionDone="Taler built",
+ command=["./build.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=False
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "microhttpd"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "testcurl"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "testzzuf"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "util"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "pq"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "curl"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "json"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("twister", "test"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "util"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "curl"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "mhd"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "pq"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "json"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "bank-lib"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchangedb"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "auditordb"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchange"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "auditor"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "lib"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchange-tools"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "testing"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "benchmark"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "backenddb"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "backend"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "lib"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "merchant-tools"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "util"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "syncdb"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "sync"))
+INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "lib"))
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="bank check",
+ description="Testing Taler Python bank",
+ descriptionDone="Done",
+ command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"],
+ workdir="../../sources/bank",
+ haltOnFailure=False,
+ env={'PYTHONUSERBASE': "$HOME/local" }
+ )
+)
+INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="wallet check",
+ description="Testing wallet-core",
+ descriptionDone="Done",
+ command=["make", "check"],
+ workdir="../../sources/wallet-core",
+ haltOnFailure=False
+ )
+)
+
+
+# FIXME: INTEGRATIONTEST_FACTORY is incomplete at this time: we still need
+# to do the actual 'make check' steps (in a jail!)
+
+
+LCOV_FACTORY = create_factory_with_deployment()
+LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
+LCOV_FACTORY.addStep(
+ ShellCommand(
+ name="fetch",
+ description="Running yarn install of wallet",
+ descriptionDone="Correctly installed",
+ command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"],
+ workdir="build/"
+ )
+)
+LCOV_FACTORY.addStep(
+ ShellCommand(
+ name="build",
+ description="Building other Taler components",
+ descriptionDone="Taler built",
+ command=["./build.sh"],
+ workdir="../../deployment/buildbot",
+ env={'TALER_COVERAGE': "1"}
+ )
+)
+LCOV_FACTORY.addStep(
+ ShellCommand(
+ name="coverage generation",
+ description="running tests",
+ descriptionDone="generating HTML report",
+ command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "./coverage.sh"],
+ workdir="../../deployment/buildbot",
+ )
+)
+
+AUDITOR_FACTORY_TEST = create_factory_with_deployment()
+AUDITOR_FACTORY_TEST.addStep(
+ ShellCommand(
+ name="Auditor reports generator",
+ description="Generating auditor reports.",
+ descriptionDone="Auditor reports correctly generated.",
+ command=["./make_auditor_reports.sh"],
+ workdir="../../deployment/buildbot",
+ env={'TALER_HOME': "/home/taler-test/"}
+ )
+)
+
+AUDITOR_FACTORY_DEMO = create_factory_with_deployment()
+AUDITOR_FACTORY_DEMO.addStep(
+ ShellCommand(
+ name="Auditor reports generator",
+ description="Generating auditor reports.",
+ descriptionDone="Auditor reports correctly generated.",
+ command=["./make_auditor_reports.sh"],
+ workdir="../../deployment/buildbot",
+ env={'TALER_HOME': "/home/demo/active-home/"}
+ )
+)
+
+TIP_RESERVE_TOPPER_FACTORY = create_factory_with_deployment()
+TIP_RESERVE_TOPPER_FACTORY.addStep(
+ ShellCommand(
+ name="tip reserve topper",
+ description="Topping the tip reserve.",
+ descriptionDone="Tip reserve has been topped.",
+ command=["./top_reserve.sh"],
+ workdir="../../deployment/buildbot"
+ )
+)
+
+BUILD_FACTORY = create_factory_with_deployment()
+BUILD_FACTORY.addStep(
+ ShellCommand(
+ name="build",
+ description="Building all Taler codebase.",
+ descriptionDone="Taler built.",
+ command=["./build.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True
+ )
+)
+
+BUILD_FACTORY.addStep(
+ ShellCommand(
+ name="config",
+ description="Generating configuration file.",
+ descriptionDone="Configuration file generated.",
+ command=["./config.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True
+ )
+)
+
+BUILD_FACTORY.addStep(
+ ShellCommand(
+ name="keys generation and sign",
+ description="Generating exchange keys, and auditor-sign them.",
+ descriptionDone="Exchange keys generated, and auditor-signed.",
+ command=["./keys.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True,
+ env={'BRANCH': util.Property("branch")}
+ )
+)
+
+BUILD_FACTORY.addStep(
+ ShellCommand(
+ name="restart services",
+ description="Restarting inactive blue-green party.",
+ descriptionDone="Restarting Taler.",
+ command=["./restart.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True,
+ env={'BRANCH': util.Property("branch")}
+ )
+)
+
+BUILD_FACTORY.addStep(
+ ShellCommand(
+ name="check services correctly restarted",
+ description="Checking services are correctly restarted.",
+ descriptionDone="All services are correctly restarted.",
+ command=["./checks.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True,
+ env={'DEPLOYMENT': "test"}
+ )
+)
+
+SELENIUM_FACTORY = create_factory_with_deployment()
+SELENIUM_FACTORY.addStep(
+ ShellCommand(
+ name="selenium",
+ description="Headless browser test",
+ descriptionDone="Test finished",
+ command=["launch_selenium_test"],
+ env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"}
+ )
+)
+
+DOC_FACTORY = create_factory_with_deployment()
+DOC_FACTORY.addStep(
+ ShellCommand(
+ name="build docs",
+ description="Building documentation",
+ descriptionDone="Documentation built.",
+ command=["./build-docs.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True
+ )
+)
+
+SITES_FACTORY = create_factory_with_deployment()
+SITES_FACTORY.addStep(
+ ShellCommand(
+ name="build Web sites",
+ description="Building all the Taler homepages",
+ descriptionDone="Sites built.",
+ command=["./build-sites.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True
+ )
+)
+
+DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment()
+DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep(
+ ShellCommand(
+ name="demo services checker",
+ description="Checking demo services are online",
+ descriptionDone="Demo services are online!.",
+ command=["./checks.sh"],
+ workdir="../../deployment/buildbot",
+ haltOnFailure=True,
+ # Needed to test the 'demo' deployment.
+ env={"DEPLOYMENT": "demo"}
+ )
+)
+
+
+##### BUILDERS DEFINED
+
+# Builder to update the buildmaster itself.
+BUILDMASTER_BUILDER = util.BuilderConfig(
+ name="buildmaster-builder",
+ workernames=["buildmaster-worker"],
+ factory=BUILDMASTER_FACTORY
+)
+
+# This builder is NOT ACTIVE!
+LINT_BUILDER = util.BuilderConfig(
+ name="lint-builder",
+ workernames=["lint-worker"],
+ factory=LINT_FACTORY
+)
+
+LCOV_BUILDER = util.BuilderConfig(
+ name="lcov-builder",
+ workernames=["lcov-worker"],
+ factory=LCOV_FACTORY
+)
+
+CHECKER_BUILDER = util.BuilderConfig(
+ name="checker-builder",
+ workernames=["checker-worker"],
+ factory=INTEGRATIONTEST_FACTORY
+)
+
+AUDITOR_BUILDER_TEST = util.BuilderConfig(
+ name="auditor-builder-test",
+ workernames=["test-auditor-worker"],
+ factory=AUDITOR_FACTORY_TEST
+)
+
+AUDITOR_BUILDER_DEMO = util.BuilderConfig(
+ name="auditor-builder-demo",
+ workernames=["demo-auditor-worker"],
+ factory=AUDITOR_FACTORY_DEMO
+)
+
+TIP_RESERVE_TOPPER_BUILDER_DEMO = util.BuilderConfig(
+ name="tip-reserve-topper-builder-demo",
+ workernames=["demo-topper-worker"],
+ factory=TIP_RESERVE_TOPPER_FACTORY
+)
+
+TIP_RESERVE_TOPPER_BUILDER_TEST = util.BuilderConfig(
+ name="tip-reserve-topper-builder-test",
+ workernames=["test-topper-worker"],
+ factory=TIP_RESERVE_TOPPER_FACTORY
+)
+
+DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig(
+ name="demo-services-checker-builder",
+ workernames="demo-worker",
+ factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY
+)
+
+TEST_BUILDER = util.BuilderConfig(
+ name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY
+)
+
+SELENIUM_BUILDER = util.BuilderConfig(
+ name="selenium-builder",
+ workernames=["selenium-worker"],
+ factory=SELENIUM_FACTORY
+)
+
+DOC_BUILDER = util.BuilderConfig(
+ name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY
+)
+
+SITES_BUILDER = util.BuilderConfig(
+ name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY
+)
+
+WALLET_BUILDER = util.BuilderConfig(
+ name="wallet-builder",
+ workernames=["wallet-worker"],
+ factory=WALLET_FACTORY
+)
+
+TALER_TEST_HEALTHCHECK_BUILDER = util.BuilderConfig(
+ name="taler-test-healthcheck-builder",
+ workernames=["taler-test-healthcheck"],
+ factory=TALER_TEST_HEALTHCHECK_FACTORY
+)
+
+TALER_DEMO_HEALTHCHECK_BUILDER = util.BuilderConfig(
+ name="taler-demo-healthcheck-builder",
+ workernames=["taler-demo-healthcheck"],
+ factory=TALER_DEMO_HEALTHCHECK_FACTORY
+)
+
+c["builders"] = [
+ LINKCHECKER_BUILDER,
+ BUILDSLAVETEST_BUILDER,
+ BUILDMASTER_BUILDER,
+ LCOV_BUILDER,
+ CHECKER_BUILDER,
+ AUDITOR_BUILDER_TEST,
+ AUDITOR_BUILDER_DEMO,
+ TEST_BUILDER,
+ TIP_RESERVE_TOPPER_BUILDER_DEMO,
+ TIP_RESERVE_TOPPER_BUILDER_TEST,
+ DEMO_SERVICES_CHECKER_BUILDER,
+ DOC_BUILDER,
+ SITES_BUILDER,
+ WALLET_BUILDER,
+ TALER_DEMO_HEALTHCHECK_BUILDER,
+ TALER_TEST_HEALTHCHECK_BUILDER,
+]
+
+####### BUILDBOT SERVICES
+
+# 'services' is a list of BuildbotService items like reporter
+# targets. The status of each build will be pushed to these
+# targets. buildbot/reporters/*.py has a variety to choose from,
+# like IRC bots.
+
+IRC = reporters.IRC(
+ "irc.eu.freenode.net",
+ "taler-bb",
+ useColors=False,
+ channels=[{
+ "channel": "#taler"
+ }],
+ password="taler-bb-pass19",
+ notify_events={
+ 'exception': 1,
+ 'successToFailure': 1,
+ 'failureToSuccess': 1
+ }
+)
+
+EMAIL = reporters.MailNotifier(
+ fromaddr="testbuild@taler.net",
+ sendToInterestedUsers=False,
+ # notify from pass to fail, and viceversa.
+ mode=("change"),
+ builders=(
+ "lcov-builder",
+ "checker-builder",
+ "doc-builder",
+ "test-builder",
+ "wallet-builder",
+ ##
+ # Rarely/never used, excluding.
+ # "selenium-builder"
+ ),
+ extraRecipients=["buildfailures@taler.net"],
+ subject="Taler build."
+)
+
+c["services"] = [IRC, EMAIL]
+
+####### PROJECT IDENTITY
+
+c["title"] = "GNU Taler"
+c["titleURL"] = "https://taler.net"
+
+# We use nginx to expose the BB under this URL.
+c["buildbotURL"] = "https://buildbot.taler.net/"
+
+
+BUILDER_LIST = [
+ "linkchecker-builder",
+ "buildslavetest-builder",
+ "buildmaster-builder",
+ "doc-builder",
+ "lcov-builder",
+ "checker-builder",
+ "auditor-builder",
+ "auditor-builder-test",
+ "auditor-builder-demo",
+ "demo-services-checker-builder",
+ "tip-reserve-topper-builder-demo",
+ "tip-reserve-topper-builder-test",
+ "sites-builder",
+ "test-builder",
+ "wallet-builder",
+ "taler-demo-healthcheck-builder",
+ "taler-test-healthcheck-builder",
+]
+
+# Authentication for Web UI.
+exec(compile(open('../taler_bb_userpass_db/taler_bb_userpass_db/db.py').read(), filename='db.py', mode='exec'))
+
+authz = util.Authz(
+ allowRules=[
+ util.ForceBuildEndpointMatcher(role="admins",
+ builder=b)
+ for b in BUILDER_LIST
+ ] + [
+ util.StopBuildEndpointMatcher(role="admins",
+ builder=b)
+ for b in BUILDER_LIST
+ ] + [
+ util.RebuildBuildEndpointMatcher(role="admins",
+ builder=b)
+ for b in BUILDER_LIST
+ ] + [
+ util.ForceBuildEndpointMatcher(role="norole",
+ builder=b)
+ for b in BUILDER_LIST
+ ] + [
+ util.StopBuildEndpointMatcher(role="norole",
+ builder=b)
+ for b in BUILDER_LIST
+ ] + [
+ util.RebuildBuildEndpointMatcher(role="norole",
+ builder=b)
+ for b in BUILDER_LIST
+ ],
+ roleMatchers=[
+ util.RolesFromUsername(
+ roles=["admins"],
+ usernames=["marcello", "florian", "christian", "buckE"]
+ )
+ ]
+)
+
+# minimalistic config to activate new web UI
+# -- formerly commented out as not packaged properly in Debian and others, see
+# https://bugzilla.redhat.com/show_bug.cgi?id=1557687
+# -- un-commented as "pip install buildbot.www" performed from buildbot-master account
+c["www"] = {
+ "port": 8010,
+ "plugins": {
+ "waterfall_view": {},
+ "console_view": {}
+ },
+ "allowed_origins": ["https://*.taler.net"],
+ "avatar_methods": [],
+ "auth": util.UserPasswordAuth(USER_PASSWORD_DB),
+ "authz": authz
+}
+
+####### DB URL
+
+c["db"] = {
+ # This specifies what database buildbot uses to store its
+ # state. You can leave this at its default for all but the
+ # largest installations.
+ "db_url": "sqlite:///state.sqlite",
+}
