diff options
author | Christian Grothoff <christian@grothoff.org> | 2021-04-21 15:20:03 +0200 |
---|---|---|
committer | Christian Grothoff <christian@grothoff.org> | 2021-04-21 15:20:03 +0200 |
commit | 95e66d13f421402cc37416abe0cb919823121d64 (patch) | |
tree | 41ab231160f24616e3daff7b59433d09a5a78a53 /buildbot | |
parent | d890abbf2f7c452f2c8f92c79a2a67464be0d3a5 (diff) | |
download | deployment-95e66d13f421402cc37416abe0cb919823121d64.tar.gz deployment-95e66d13f421402cc37416abe0cb919823121d64.tar.bz2 deployment-95e66d13f421402cc37416abe0cb919823121d64.zip |
major code cleanup for buildbot configuration file (no semantic change intended)
Diffstat (limited to 'buildbot')
-rw-r--r-- | buildbot/master.cfg | 1380 | ||||
-rw-r--r-- | buildbot/master.cfg-GivingUpNewFormatting | 1159 | ||||
-rw-r--r-- | buildbot/master.cfg.BACK-movingFunctionsAround | 1153 |
3 files changed, 672 insertions, 3020 deletions
diff --git a/buildbot/master.cfg b/buildbot/master.cfg index 9ae4927..d679268 100644 --- a/buildbot/master.cfg +++ b/buildbot/master.cfg @@ -1,6 +1,6 @@ ## # This file is part of TALER -# (C) 2016-2020 Taler Systems SA +# (C) 2016-2021 Taler Systems SA # # TALER is free software; you can redistribute it and/or # modify it under the terms of the GNU Affero General Public @@ -29,296 +29,50 @@ from buildbot.reporters.generators.build import BuildStatusGenerator # This is a sample buildmaster config file. It must be # installed as 'master.cfg' in your buildmaster's base -# directory. This is the dictionary that the buildmaster -# pays attention to. We also use a shorter alias to save -# typing. -c = BuildmasterConfig = {} - -# Silence warning and allow very basic phoning home. -c["buildbotNetUsageData"] = "basic" - - -####### WORKERS +# directory. # The 'workers' list defines the set of recognized workers. # Each element is a Worker object, specifying a unique worker # name and password. The same worker name and password must # be configured on the worker. -c["workers"] = [ - - ## - # This worker restarts the buildmaster itself on - # changes to this file. - # Location: /home/buildbot-master @ taler.net - worker.Worker("buildmaster-worker", "buildmaster-pass"), - - ## - # This worker makes the code coverage and publishes it - # under the "lcov" Website. - worker.Worker("lcov-worker", "lcov-pass"), - - ## - # This worker builds everything and runs our 'make check' - # test suite against 'everything'. - worker.Worker("checker-worker", "checker-pass"), - - ## - # This worker builds manuals / API docs / tutorials. - # Location: /home/docbuilder @ taler.net - worker.Worker("doc-worker", "doc-pass"), - - ## - # This worker builds Websites: www and stage. - # - worker.Worker("sites-worker", "sites-pass"), - - ## - # This worker builds Taler for the 'test' deployment. - worker.Worker("test-worker", "test-pass"), - - ## - # This worker compiles the auditor reports for the "green" - # demo deployment. - worker.Worker("test-auditor-worker", "test-auditor-pass"), - worker.Worker("demo-auditor-worker", "demo-auditor-pass"), - - ## - # This worker checks that all the services run under the - # 'demo' deployment are up&running. - worker.Worker("demo-worker", "demo-pass"), - - ## - # This worker builds wallet-core. - worker.Worker("wallet-worker", "wallet-pass"), - - ## - # health checks performed by wallet-cli for demo - worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass"), - - ## - # health checks performed by wallet-cli for test - worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass"), - - ## - # testing buildbot using the "buildslavetest" user (for no specific reason except it exists) - # Location: /home/buidlslavetest @ taler.net - worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo"), - - ## - # linkchecker worker checks for dead links in the Website - # Location: /home/linkchecker @ taler.net - worker.Worker("linkchecker-worker", "linkchecker-pass"), - - ## - # codespell worker checks for spelling mistakes in code - # Location: /home/codespell @ taler.net - worker.Worker("codespell-worker", "codespell-pass") -] - -# 'protocols' contains information about protocols which master -# will use for communicating with workers. You must define at -# least 'port' option that workers could connect to your master -# with this protocol. 'port' must match the value configured into -# the workers (with their --master option) -c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}} - -####### CHANGESOURCES - -# the 'change_source' setting tells the buildmaster how it should -# find out about source code changes. +WORKERS = [] -# NOTE: BB is bound to localhost -ALLCS = changes.PBChangeSource(user="allcs", passwd="allcs") - -c["change_source"] = [ALLCS] +# 'services' is a list of BuildbotService items like reporter +# targets. The status of each build will be pushed to these +# targets. buildbot/reporters/*.py has a variety to choose from, +# like IRC bots. +SERVICES = [] -####### SCHEDULERS +# The 'builders' list defines the Builders, which tell Buildbot +# how to perform a build: what steps, and which workers can execute +# them. Note that any particular build will only take place on +# one worker. +BUILDERS = [] -# Configure the Schedulers, which decide how to react to incoming +# Configures the Schedulers, which decide how to react to incoming # changes. +SCHEDULERS = [] -# Run at 6am every day: coverage analysis and auditor. -NIGHTLY_SCHEDULER = schedulers.Nightly( - name="nightly-scheduler", - builderNames=["lcov-builder", - "auditor-builder-demo", - "auditor-builder-test"], - branch="master", - hour=6, - minute=0 -) - -# linkchecker SCHEDULER -# Run at 7:45am every day. -LINKCHECKER_SCHEDULER = schedulers.Nightly( - name="linkchecker-scheduler", - builderNames=["linkchecker-builder"], - branch="master", - hour=7, - minute=45 -) - -# buildslavetest SCHEDULER -BUILDSLAVETEST_SCHEDULER = schedulers.SingleBranchScheduler( -# properties = { -# 'owner': ['grothoff@taler.net',] -# }, -#owners=['grothoff@taler.net',] - name="buildslavetest-scheduler", - builderNames=["buildslavetest-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", project_re="(help|deployment)" - ), - treeStableTimer=None, -) - -# Docs run if master or stable branch of 'docs' (or deployment) changed. -DOC_SCHEDULER = schedulers.SingleBranchScheduler( - name="periodic-doc-scheduler", - builderNames=["doc-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", project_re="(docs|deployment)" - ), - treeStableTimer=None, -) - - -# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed: -def twister_web_page(change): - _change = change.asDict() - repo = _change.get("project") - if repo in ["www", "buywith", "deployment"]: - return True - - files = _change.get("files") - for file in files: - if re.search(r"web", file.get("name", "")) \ - and "twister" == repo: - return True - return False - -# Sites are re-build whenever deployment, www buywith, or twister changes. -SITES_SCHEDULER = schedulers.SingleBranchScheduler( - name="sites-scheduler", - builderNames=["sites-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", - filter_fn=twister_web_page - ), - treeStableTimer=None -) - -# Buildmaster is notified whenever deployment changes -BUILDMASTER_SCHEDULER = schedulers.SingleBranchScheduler( - name="buildmaster-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(deployment)" - ), - treeStableTimer=None, - builderNames=["buildmaster-builder"] -) +# Array of builders to be scheduled every night. +NIGHTLY_BUILDERS=[] -# Wallet is re-build whenever wallet-core or deployment changes -WALLET_SCHEDULER = schedulers.SingleBranchScheduler( - name="wallet-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet-core|deployment)" - ), - treeStableTimer=None, - builderNames=["wallet-builder"] -) - -# Healthchecks are done on wallet or deployment changes to master -TALER_HEALTHCHECK_SCHEDULER = schedulers.SingleBranchScheduler( - name="taler-healthcheck-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet|deployment)" - ), - treeStableTimer=None, - builderNames=["taler-test-healthcheck-builder", - "taler-demo-healthcheck-builder"] -) - -# We check demo once per hour. -DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic( - name="demo-services-checker-scheduler", - periodicBuildTimer=60 * 60, # 1 hour - builderNames=["demo-services-checker-builder"] -) - -# Scheduler that triggers if anything changes -ALL_SCHEDULER = schedulers.SingleBranchScheduler( - name="all-scheduler", - change_filter=util.ChangeFilter( - branch_re="(master|stable)", - project_re="(backoffice|wallet-core|bank|exchange|" - "merchant|deployment|twister|sync|" - "help|taler-merchant-demos)" - ), - treeStableTimer=None, - builderNames=["test-builder", "checker-builder", "codespell-builder"] -) +# Array of builders to be scheduled whenever any of the code Git repos change +CODECHANGE_BUILDERS=[] -# Consider adding other Python parts, like the various frontends. -# NOTE: scheduler is NOT active! (commented out below) -LINT_SCHEDULER = schedulers.SingleBranchScheduler( - name="lint-scheduler", - change_filter=util.ChangeFilter( - branch="master", project_re="(bank|donations|survey|blog)" - ), - treeStableTimer=None, - builderNames=["lint-builder"] -) +# Array of builders to be scheduled whenever the wallet-core or deployment change +WALLETCHANGE_BUILDERS=[] -# Provide "force" button in the web UI. -FORCE_SCHEDULER = schedulers.ForceScheduler( - name="force-scheduler", - builderNames=[ - "buildmaster-builder", - "buildslavetest-builder", - "lcov-builder", - "checker-builder", - "codespell-builder", - "auditor-builder-test", - "auditor-builder-demo", - "test-builder", - "doc-builder", - "sites-builder", - "wallet-builder", - "taler-test-healthcheck-builder", - "taler-demo-healthcheck-builder", - ] -) +# Array of builder names for which build status reports should be sent via e-mail +EMAIL_ALERT_BUILDERS=[] -c["schedulers"] = [ - NIGHTLY_SCHEDULER, - DEMO_SERVICES_CHECKER_SCHEDULER, - DOC_SCHEDULER, - BUILDMASTER_SCHEDULER, - SITES_SCHEDULER, - ALL_SCHEDULER, - FORCE_SCHEDULER, - WALLET_SCHEDULER, - TALER_HEALTHCHECK_SCHEDULER, - BUILDSLAVETEST_SCHEDULER, - LINKCHECKER_SCHEDULER - - ## - # Rarely/never used, excluding. - # DEBUG_SCHEDULER, - # LINT_SCHEDULER -] - -####### BUILDERS -# The 'builders' list defines the Builders, which tell Buildbot -# how to perform a build: what steps, and which workers can execute -# them. Note that any particular build will only take place on -# one worker. +############Convenience functions ################# +# Create a FACTORY with a deployment.git checkout as the first step. +def create_factory_with_deployment(): + f = util.BuildFactory() + update_deployment (f) + return f # Convenience function that checks out a Git repository. # First argument is the URL of the Git to clone, second @@ -334,7 +88,6 @@ def git_step(repo,target_branch="master"): branch=target_branch ) - # Convenience function that runs 'make check' in a # directory of the code inside of a netjail. def jailed_check(package,srcdir): @@ -363,13 +116,15 @@ def update_deployment(factory): workdir="../.." )) -# Create a FACTORY with a deployment.git checkout as the first step. -def create_factory_with_deployment(): - f = util.BuildFactory() - update_deployment (f) - return f -# Factory to update the buildmaster itself. +################ 1: BUILDMASTER JOB ################################### + +## +# This worker restarts the buildmaster itself on +# changes to this file. +# Location: /home/buildbot-master @ taler.net +WORKERS.append(worker.Worker("buildmaster-worker", "buildmaster-pass")) + BUILDMASTER_FACTORY = create_factory_with_deployment() BUILDMASTER_FACTORY.addStep( ShellCommand( @@ -381,166 +136,180 @@ BUILDMASTER_FACTORY.addStep( ) ) -# linkchecker FACTORY -LINKCHECKER_FACTORY = create_factory_with_deployment() -LINKCHECKER_FACTORY.addStep( +BUILDERS.append(util.BuilderConfig( + name="buildmaster-builder", + workernames=["buildmaster-worker"], + factory=BUILDMASTER_FACTORY +)) + +EMAIL_ALERT_BUILDERS.append("buildmaster-builder") + +# Buildmaster is notified whenever deployment.git changes +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="buildmaster-scheduler", + change_filter=util.ChangeFilter( + branch="master", + project_re="(deployment)" + ), + treeStableTimer=None, + builderNames=["buildmaster-builder"] +)) + + +################ 2: DOCUMENTATION JOB ################################### + +## +# This worker builds manuals / API docs / tutorials. +# Location: /home/docbuilder @ taler.net +WORKERS.append(worker.Worker("doc-worker", "doc-pass")) + +DOC_FACTORY = create_factory_with_deployment() +DOC_FACTORY.addStep( ShellCommand( - name="linkchecker", - description="Check taler.net website for broken links && Notify", - descriptionDone="Results of wget in buildbot logs.", - command=["/home/linkchecker/deployment/buildbot/linkchecker.sh"], - workdir="/home/linkchecker", + name="build docs", + description="Building documentation", + descriptionDone="Documentation built.", + command=["./build-docs.sh"], + workdir="../../deployment/buildbot", haltOnFailure=True ) ) -# buildslavetest FACTORY -BUILDSLAVETEST_FACTORY = create_factory_with_deployment() -BUILDSLAVETEST_FACTORY.addStep( - ShellCommand( - name="buildslavetest script (for testing purposes)", - description="Build Slave Test", - descriptionDone="buildslavetest: Done", - command=["./buildslavetest.sh"], - workdir="/home/buildslavetest/" - ) -) +BUILDERS.append(util.BuilderConfig( + name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY +)) -TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) +EMAIL_ALERT_BUILDERS.append("doc-builder") -TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( +#sphinxErrorNotifier = reporters.MailNotifier( +# fromaddr="bb@taler.net", +# sendToInterestedUsers=False, +# addLogs=['build docs.stdio',], +# useTls=True, +# # notify if sphinx exits with error (command line option in Makefile turns warnings into exit 1) +# mode=('failing'), +# builders=('doc-builder',), +# extraRecipients=['sphinxerrors@taler.net'] +#) + + +# Docs run if master or stable branch of 'docs' (or deployment) changed. +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="periodic-doc-scheduler", + builderNames=["doc-builder"], + change_filter=util.ChangeFilter( + branch_re="(master|stable)", project_re="(docs|deployment)" + ), + treeStableTimer=None, +)) + + +################ 3: WEBSITE JOB ################################### + +## +# This worker builds Websites: www and stage. +# +WORKERS.append(worker.Worker("sites-worker", "sites-pass")) + +SITES_FACTORY = create_factory_with_deployment() +SITES_FACTORY.addStep( ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", + name="build Web sites", + description="Building all the Taler homepages", + descriptionDone="Sites built.", + command=["./build-sites.sh"], + workdir="../../deployment/buildbot", + haltOnFailure=True ) ) +BUILDERS.append(util.BuilderConfig( + name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY +)) -WALLET_FACTORY = create_factory_with_deployment() -WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -WALLET_FACTORY.addStep( +EMAIL_ALERT_BUILDERS.append("sites-builder") + +# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed: +def twister_web_page(change): + _change = change.asDict() + repo = _change.get("project") + if repo in ["www", "buywith", "deployment"]: + return True + + files = _change.get("files") + for file in files: + if re.search(r"web", file.get("name", "")) \ + and "twister" == repo: + return True + return False + +# Sites are re-build whenever deployment, www buywith, or twister changes. +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="sites-scheduler", + builderNames=["sites-builder"], + change_filter=util.ChangeFilter( + branch_re="(master|stable)", + filter_fn=twister_web_page + ), + treeStableTimer=None +)) + + +################ 4: LCOV JOB ################################### + +## +# This worker makes the code coverage and publishes it +# under the "lcov" Website. +WORKERS.append(worker.Worker("lcov-worker", "lcov-pass")) + +LCOV_FACTORY = create_factory_with_deployment() +LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) +LCOV_FACTORY.addStep( ShellCommand( name="fetch", - description="Running yarn install", + description="Running yarn install of wallet", descriptionDone="Correctly installed", command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], workdir="build/" ) ) -WALLET_FACTORY.addStep( - ShellCommand( - name="test", - description="Running wallet tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose"], - workdir="build/", - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -WALLET_FACTORY.addStep( +LCOV_FACTORY.addStep( ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-s", "TESTKUDOS:4"], - workdir="build/", + name="build", + description="Building other Taler components", + descriptionDone="Taler built", + command=["./build.sh"], + workdir="../../deployment/buildbot", + env={'TALER_COVERAGE': "1"} ) ) -# WALLET_FACTORY.addStep( -# ShellCommand( -# name="lint", -# description="Linting the wallet", -# descriptionDone="Linting done", -# command=["make", "lint"], -# workdir="build/" -# ) -# ) - -DEBUG_FACTORY = util.BuildFactory() -DEBUG_FACTORY.addStep( +LCOV_FACTORY.addStep( ShellCommand( - name="echo debug", - description="just echoing a word", - descriptionDone="builder responded", - command=["echo", "I'm here!"] + name="coverage generation", + description="running tests", + descriptionDone="generating HTML report", + command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "/home/lcovworker/deployment/buildbot/with-postgres.sh","./coverage.sh"], + workdir="../../deployment/buildbot", + env={'PATH': "${HOME}/local/bin:${PATH}"} ) ) +BUILDERS.append(util.BuilderConfig( + name="lcov-builder", + workernames=["lcov-worker"], + factory=LCOV_FACTORY +)) -def lint_dispatcher(project): - return "./lint_%s.sh" % project +EMAIL_ALERT_BUILDERS.append("lcov-builder") +NIGHTLY_BUILDERS.append("lcov-builder") -LINT_FACTORY = util.BuildFactory() -LINT_FACTORY.addStep( - ShellCommand( - name="Python linter", - description="linting Python", - descriptionDone="linting done", - command=util.Transform(lint_dispatcher, util.Property("project")), - workdir="../../deployment/taler-build" - ) -) +################ 5: UNIT TEST JOB ################################### +## +# This worker builds everything and runs our 'make check' +# test suite against 'everything'. +WORKERS.append(worker.Worker("checker-worker", "checker-pass")) INTEGRATIONTEST_FACTORY = create_factory_with_deployment() INTEGRATIONTEST_FACTORY.addStep( @@ -673,6 +442,241 @@ INTEGRATIONTEST_FACTORY.addStep( # to do the actual 'make check' steps (in a jail!) +BUILDERS.append(util.BuilderConfig( + name="checker-builder", + workernames=["checker-worker"], + factory=INTEGRATIONTEST_FACTORY +)) + +EMAIL_ALERT_BUILDERS.append("checker-builder") +CODECHANGE_BUILDERS.append("checker-builder") + + +################ 6: 'test.taler.net' deployment JOB ################################### + +## +# This worker builds Taler for the 'test' deployment. +WORKERS.append(worker.Worker("test-worker", "test-pass")) + +# buildslavetest FACTORY +BUILDSLAVETEST_FACTORY = create_factory_with_deployment() +BUILDSLAVETEST_FACTORY.addStep( + ShellCommand( + name="buildslavetest script (for testing purposes)", + description="Build Slave Test", + descriptionDone="buildslavetest: Done", + command=["./buildslavetest.sh"], + workdir="/home/buildslavetest/" + ) +) + +# buildslavetest BUILDER +BUILDERS.append(util.BuilderConfig( + name="buildslavetest-builder", + workernames=["buildslavetest-worker"], + factory=BUILDSLAVETEST_FACTORY +)) + +EMAIL_ALERT_BUILDERS.append("buildslavetest-builder") + +# buildslavetest SCHEDULER +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="buildslavetest-scheduler", + builderNames=["buildslavetest-builder"], + change_filter=util.ChangeFilter( + branch_re="(master|stable)", project_re="(help|deployment)" + ), + treeStableTimer=None, +)) + +################ 7: 'test-auditor' deployment JOB ################################### + + +## +# This worker compiles the auditor reports for the "test" +# demo deployment. +WORKERS.append(worker.Worker("test-auditor-worker", "test-auditor-pass")) + +AUDITOR_FACTORY_TEST = create_factory_with_deployment() +AUDITOR_FACTORY_TEST.addStep( + ShellCommand( + name="Auditor reports generator", + description="Generating auditor reports.", + descriptionDone="Auditor reports correctly generated.", + command=["./make_auditor_reports.sh"], + workdir="../../deployment/buildbot", + env={'TALER_HOME': "/home/taler-test/"} + ) +) + +BUILDERS.append(util.BuilderConfig( + name="auditor-builder-test", + workernames=["test-auditor-worker"], + factory=AUDITOR_FACTORY_TEST +)) +NIGHTLY_BUILDERS.append("auditor-builder-test") + + +################ 8: 'demo-auditor' deployment JOB ################################### +## +# This worker compiles the auditor reports for the "green" +# demo deployment. +WORKERS.append(worker.Worker("demo-auditor-worker", "demo-auditor-pass")) + +AUDITOR_FACTORY_DEMO = create_factory_with_deployment() +AUDITOR_FACTORY_DEMO.addStep( + ShellCommand( + name="Auditor reports generator", + description="Generating auditor reports.", + descriptionDone="Auditor reports correctly generated.", + command=["./make_auditor_reports.sh"], + workdir="../../deployment/buildbot", + env={'TALER_HOME': "/home/demo/active-home/"} + ) +) + +BUILDERS.append(util.BuilderConfig( + name="auditor-builder-demo", + workernames=["demo-auditor-worker"], + factory=AUDITOR_FACTORY_DEMO +)) + +NIGHTLY_BUILDERS.append("auditor-builder-demo") + + +################ 8: 'build wallet-core' JOB ################################### + +## +# This worker builds wallet-core. +WORKERS.append(worker.Worker("wallet-worker", "wallet-pass")) + +WALLET_FACTORY = create_factory_with_deployment() +WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) +WALLET_FACTORY.addStep( + ShellCommand( + name="fetch", + description="Running yarn install", + descriptionDone="Correctly installed", + command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], + workdir="build/" + ) +) +WALLET_FACTORY.addStep( + ShellCommand( + name="test", + description="Running wallet tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose"], + workdir="build/", + ) +) +WALLET_FACTORY.addStep( + ShellCommand( + name="test-withdraw", + description="Running wallet withdraw tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-w", "TESTKUDOS:10"], + workdir="build/", + ) +) +WALLET_FACTORY.addStep( + ShellCommand( + name="test-spend", + description="Running wallet spend tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-s", "TESTKUDOS:4"], + workdir="build/", + ) +) +# WALLET_FACTORY.addStep( +# ShellCommand( +# name="lint", +# description="Linting the wallet", +# descriptionDone="Linting done", +# command=["make", "lint"], +# workdir="build/" +# ) +# ) + +BUILDERS.append(util.BuilderConfig( + name="wallet-builder", + workernames=["wallet-worker"], + factory=WALLET_FACTORY +)) + +EMAIL_ALERT_BUILDERS.append("wallet-builder") + +# Wallet is re-build whenever wallet-core or deployment changes +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="wallet-scheduler", + change_filter=util.ChangeFilter( + branch="master", + project_re="(wallet-core|deployment)" + ), + treeStableTimer=None, + builderNames=["wallet-builder"] +)) + + +################ 9: 'check links' JOB ################################### + +## +# linkchecker worker checks for dead links in the Website +# Location: /home/linkchecker @ taler.net +WORKERS.append(worker.Worker("linkchecker-worker", "linkchecker-pass")) + +# linkchecker FACTORY +LINKCHECKER_FACTORY = create_factory_with_deployment() +LINKCHECKER_FACTORY.addStep( + ShellCommand( + name="linkchecker", + description="Check taler.net website for broken links && Notify", + descriptionDone="Results of wget in buildbot logs.", + command=["/home/linkchecker/deployment/buildbot/linkchecker.sh"], + workdir="/home/linkchecker", + haltOnFailure=True + ) +) + +# linkchecker BUILDER +# worker at linkchecker@taler.net +BUILDERS.append(util.BuilderConfig( + name="linkchecker-builder", + workernames="linkchecker-worker", + factory=LINKCHECKER_FACTORY +)) + +docs_generator = BuildStatusGenerator( + mode=('change','problem','failing','exception',), + builders=[ + 'linkchecker-builder', + ], + message_formatter=reporters.MessageFormatter( + template_type='plain', + wantSteps=True, + wantLogs=True + ), + add_logs=True + ) + +SERVICES.append(reporters.MailNotifier( + fromaddr="bb@taler.net", + generators=[docs_generator], + sendToInterestedUsers=False, + useTls=True, + extraRecipients=['linkcheck@taler.net'] +)) + +NIGHTLY_BUILDERS.append("linkchecker-builder") + + +################ 10: 'check spelling' JOB ################################### + +## +# codespell worker checks for spelling mistakes in code +# Location: /home/codespell @ taler.net +WORKERS.append(worker.Worker("codespell-worker", "codespell-pass")) + CODESPELL_FACTORY = create_factory_with_deployment() CODESPELL_FACTORY.addStep( ShellCommand( @@ -720,8 +724,6 @@ CODESPELL_FACTORY.addStep( workdir="/home/codespell/sources/" ) ) - - CODESPELL_FACTORY.addStep( ShellCommand( name="gana", @@ -731,7 +733,6 @@ CODESPELL_FACTORY.addStep( workdir="/home/codespell/sources/exchange/" ) ) - CODESPELL_FACTORY.addStep( ShellCommand( name="doxygen::exchange", @@ -741,7 +742,6 @@ CODESPELL_FACTORY.addStep( workdir="/home/codespell/sources/exchange/doc/doxygen" ) ) - CODESPELL_FACTORY.addStep( ShellCommand( name="tag", @@ -751,7 +751,6 @@ CODESPELL_FACTORY.addStep( workdir="/home/codespell/sources/" ) ) - CODESPELL_FACTORY.addStep( ShellCommand( name="tag", @@ -761,7 +760,6 @@ CODESPELL_FACTORY.addStep( workdir="/home/codespell/sources/" ) ) - CODESPELL_FACTORY.addStep( ShellCommand( name="doxygen::merchant", @@ -771,66 +769,145 @@ CODESPELL_FACTORY.addStep( workdir="/home/codespell/sources/merchant/doc/doxygen" ) ) +BUILDERS.append(util.BuilderConfig( + name="codespell-builder", + workernames=["codespell-worker"], + factory=CODESPELL_FACTORY +)) +CODECHANGE_BUILDERS.append("codespell-builder") +################ 11: 'demo checks' JOB ################################### +## +# This worker checks that all the services run under the +# 'demo' deployment are up&running. +WORKERS.append(worker.Worker("demo-worker", "demo-pass")) -LCOV_FACTORY = create_factory_with_deployment() -LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -LCOV_FACTORY.addStep( +DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment() +DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep( + ShellCommand( + name="demo services checker", + description="Checking demo services are online", + descriptionDone="Demo services are online!.", + command=["./checks.sh"], + workdir="../../deployment/buildbot", + haltOnFailure=True, + # Needed to test the 'demo' deployment. + env={"DEPLOYMENT": "demo"} + ) +) +BUILDERS.append(util.BuilderConfig( + name="demo-services-checker-builder", + workernames="demo-worker", + factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY +)) +EMAIL_ALERT_BUILDERS.append("demo-services-checker-builder") + +# We check demo once per hour. +SCHEDULERS.append(schedulers.Periodic( + name="demo-services-checker-scheduler", + periodicBuildTimer=60 * 60, # 1 hour + builderNames=["demo-services-checker-builder"] +)) + + +################ 12: 'demo health wallet-cli check' JOB ################################### + + +## +# health checks performed by wallet-cli for demo +WORKERS.append(worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass")) + +TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment() +TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) +TALER_DEMO_HEALTHCHECK_FACTORY.addStep( ShellCommand( name="fetch", - description="Running yarn install of wallet", + description="Running yarn install", descriptionDone="Correctly installed", command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], workdir="build/" ) ) -LCOV_FACTORY.addStep( +TALER_DEMO_HEALTHCHECK_FACTORY.addStep( ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - env={'TALER_COVERAGE': "1"} + name="test-withdraw", + description="Running wallet withdraw tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-w", "TESTKUDOS:10"], + workdir="build/", ) ) -LCOV_FACTORY.addStep( +TALER_DEMO_HEALTHCHECK_FACTORY.addStep( ShellCommand( - name="coverage generation", - description="running tests", - descriptionDone="generating HTML report", - command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "/home/lcovworker/deployment/buildbot/with-postgres.sh","./coverage.sh"], - workdir="../../deployment/buildbot", - env={'PATH': "${HOME}/local/bin:${PATH}"} + name="test-spend", + description="Running wallet spend tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-s", "TESTKUDOS:4"], + workdir="build/", ) ) +BUILDERS.append(util.BuilderConfig( + name="taler-demo-healthcheck-builder", + workernames=["taler-demo-healthcheck"], + factory=TALER_DEMO_HEALTHCHECK_FACTORY +)) -AUDITOR_FACTORY_TEST = create_factory_with_deployment() -AUDITOR_FACTORY_TEST.addStep( +WALLETCHANGE_BUILDERS.append("taler-demo-healthcheck-builder") + +################ 13: 'test health wallet-cli check' JOB ################################### + +## +# health checks performed by wallet-cli for test +WORKERS.append(worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass")) + +TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment() +TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) +TALER_TEST_HEALTHCHECK_FACTORY.addStep( ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/taler-test/"} - ) + name="fetch", + description="Running yarn install", + descriptionDone="Correctly installed", + command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], + workdir="build/" + ) ) - -AUDITOR_FACTORY_DEMO = create_factory_with_deployment() -AUDITOR_FACTORY_DEMO.addStep( +TALER_TEST_HEALTHCHECK_FACTORY.addStep( ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/demo/active-home/"} - ) + name="test-withdraw", + description="Running wallet withdraw tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"], + workdir="build/", + ) ) +TALER_TEST_HEALTHCHECK_FACTORY.addStep( + ShellCommand( + name="test-spend", + description="Running wallet spend tests", + descriptionDone="Test correctly run", + command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"], + workdir="build/", + ) +) +BUILDERS.append(util.BuilderConfig( + name="taler-test-healthcheck-builder", + workernames=["taler-test-healthcheck"], + factory=TALER_TEST_HEALTHCHECK_FACTORY +)) + + +WALLETCHANGE_BUILDERS.append("taler-test-healthcheck-builder") + + +################ 14: upgrade test deployment JOB ################################### + +## +# testing buildbot using the "buildslavetest" user (for no specific reason except it exists) +# Location: /home/buidlslavetest @ taler.net +WORKERS.append(worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo")) BUILD_FACTORY = create_factory_with_deployment() BUILD_FACTORY.addStep( @@ -903,185 +980,92 @@ BUILD_FACTORY.addStep( ) ) -SELENIUM_FACTORY = create_factory_with_deployment() -SELENIUM_FACTORY.addStep( - ShellCommand( - name="selenium", - description="Headless browser test", - descriptionDone="Test finished", - command=["launch_selenium_test"], - env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"} - ) -) - -DOC_FACTORY = create_factory_with_deployment() -DOC_FACTORY.addStep( - ShellCommand( - name="build docs", - description="Building documentation", - descriptionDone="Documentation built.", - command=["./build-docs.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -SITES_FACTORY = create_factory_with_deployment() -SITES_FACTORY.addStep( - ShellCommand( - name="build Web sites", - description="Building all the Taler homepages", - descriptionDone="Sites built.", - command=["./build-sites.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) +BUILDERS.append(util.BuilderConfig( + name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY +)) -DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="demo services checker", - description="Checking demo services are online", - descriptionDone="Demo services are online!.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - # Needed to test the 'demo' deployment. - env={"DEPLOYMENT": "demo"} - ) -) +EMAIL_ALERT_BUILDERS.append("test-builder") +# Scheduler that triggers if anything changes +CODECHANGE_BUILDERS.append("test-builder") -##### BUILDERS DEFINED -# Builder to update the buildmaster itself. -BUILDMASTER_BUILDER = util.BuilderConfig( - name="buildmaster-builder", - workernames=["buildmaster-worker"], - factory=BUILDMASTER_FACTORY -) +################ 15: Python linting JOB ################################### -# linkchecker BUILDER -# worker at linkchecker@taler.net -LINKCHECKER_BUILDER = util.BuilderConfig( - name="linkchecker-builder", - workernames="linkchecker-worker", - factory=LINKCHECKER_FACTORY -) +def lint_dispatcher(project): + return "./lint_%s.sh" % project -# buildslavetest BUILDER -BUILDSLAVETEST_BUILDER = util.BuilderConfig( - name="buildslavetest-builder", - workernames=["buildslavetest-worker"], - factory=BUILDSLAVETEST_FACTORY +LINT_FACTORY = util.BuildFactory() +LINT_FACTORY.addStep( + ShellCommand( + name="Python linter", + description="linting Python", + descriptionDone="linting done", + command=util.Transform(lint_dispatcher, util.Property("project")), + workdir="../../deployment/taler-build" + ) ) - # This builder is NOT ACTIVE! -LINT_BUILDER = util.BuilderConfig( - name="lint-builder", - workernames=["lint-worker"], - factory=LINT_FACTORY -) +#BUILDERS.append(util.BuilderConfig( +# name="lint-builder", +# workernames=["lint-worker"], +# factory=LINT_FACTORY +#)) -LCOV_BUILDER = util.BuilderConfig( - name="lcov-builder", - workernames=["lcov-worker"], - factory=LCOV_FACTORY -) +# Consider adding other Python parts, like the various frontends. +# NOTE: scheduler is NOT active! (commented out below) +#SCHEDULERS.append(schedulers.SingleBranchScheduler( +# name="lint-scheduler", +# change_filter=util.ChangeFilter( +# branch="master", project_re="(bank|donations|survey|blog)" +# ), +# treeStableTimer=None, +# builderNames=["lint-builder"] +#)) -CHECKER_BUILDER = util.BuilderConfig( - name="checker-builder", - workernames=["checker-worker"], - factory=INTEGRATIONTEST_FACTORY -) -CODESPELL_BUILDER = util.BuilderConfig( - name="codespell-builder", - workernames=["codespell-worker"], - factory=CODESPELL_FACTORY -) +################ 16: Selenium JOB ################################### -AUDITOR_BUILDER_TEST = util.BuilderConfig( - name="auditor-builder-test", - workernames=["test-auditor-worker"], - factory=AUDITOR_FACTORY_TEST -) - -AUDITOR_BUILDER_DEMO = util.BuilderConfig( - name="auditor-builder-demo", - workernames=["demo-auditor-worker"], - factory=AUDITOR_FACTORY_DEMO -) -DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig( - name="demo-services-checker-builder", - workernames="demo-worker", - factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY -) - -TEST_BUILDER = util.BuilderConfig( - name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY +SELENIUM_FACTORY = create_factory_with_deployment() +SELENIUM_FACTORY.addStep( + ShellCommand( + name="selenium", + description="Headless browser test", + descriptionDone="Test finished", + command=["launch_selenium_test"], + env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"} + ) ) -SELENIUM_BUILDER = util.BuilderConfig( - name="selenium-builder", - workernames=["selenium-worker"], - factory=SELENIUM_FACTORY -) +#BUILDERS.append(util.BuilderConfig( +# name="selenium-builder", +# workernames=["selenium-worker"], +# factory=SELENIUM_FACTORY +#)) -DOC_BUILDER = util.BuilderConfig( - name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY -) -SITES_BUILDER = util.BuilderConfig( - name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY -) +################ 99: debug stuff JOB ################################### -WALLET_BUILDER = util.BuilderConfig( - name="wallet-builder", - workernames=["wallet-worker"], - factory=WALLET_FACTORY +DEBUG_FACTORY = util.BuildFactory() +DEBUG_FACTORY.addStep( + ShellCommand( + name="echo debug", + description="just echoing a word", + descriptionDone="builder responded", + command=["echo", "I'm here!"] + ) ) -TALER_TEST_HEALTHCHECK_BUILDER = util.BuilderConfig( - name="taler-test-healthcheck-builder", - workernames=["taler-test-healthcheck"], - factory=TALER_TEST_HEALTHCHECK_FACTORY -) -TALER_DEMO_HEALTHCHECK_BUILDER = util.BuilderConfig( - name="taler-demo-healthcheck-builder", - workernames=["taler-demo-healthcheck"], - factory=TALER_DEMO_HEALTHCHECK_FACTORY -) +####################################################################### -c["builders"] = [ - BUILDMASTER_BUILDER, - LINKCHECKER_BUILDER, - BUILDSLAVETEST_BUILDER, - LCOV_BUILDER, - CHECKER_BUILDER, - CODESPELL_BUILDER, - AUDITOR_BUILDER_TEST, - AUDITOR_BUILDER_DEMO, - TEST_BUILDER, - DEMO_SERVICES_CHECKER_BUILDER, - DOC_BUILDER, - SITES_BUILDER, - WALLET_BUILDER, - TALER_DEMO_HEALTHCHECK_BUILDER, - TALER_TEST_HEALTHCHECK_BUILDER, -] - -####### BUILDBOT SERVICES +# Compute array of the names of all of our builders +BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) -# 'services' is a list of BuildbotService items like reporter -# targets. The status of each build will be pushed to these -# targets. buildbot/reporters/*.py has a variety to choose from, -# like IRC bots. +####### GENERAL PURPOSE BUILDBOT SERVICES ####################### -IRC = reporters.IRC( +SERVICES.append(reporters.IRC( "irc.eu.freenode.net", "taler-bb", useColors=False, @@ -1094,122 +1078,102 @@ IRC = reporters.IRC( 'successToFailure': 1, 'failureToSuccess': 1 } -) +)) -mail_generator = BuildStatusGenerator( - mode=('change','problem','failing','exception',), - builders=[ - "lcov-builder", - "checker-builder", - "doc-builder", - "test-builder", - "wallet-builder", - ], - message_formatter=reporters.MessageFormatter( - template_type='plain', - wantSteps=True, - wantLogs=True - ), -# add_logs=True - ) - - -test_generator = BuildStatusGenerator( - mode=('change','problem','failing','exception',), - builders=[ - 'buildslavetest-builder', - ], - message_formatter=reporters.MessageFormatter( - template_type='plain', - wantSteps=True, - wantLogs=True - ), -# add_logs=True - ) - -docs_generator = BuildStatusGenerator( - mode=('change','problem','failing','exception',), - builders=[ - 'linkchecker-builder', - ], - message_formatter=reporters.MessageFormatter( - template_type='plain', - wantSteps=True, - wantLogs=True - ), -# add_logs=True - ) - -EMAIL = reporters.MailNotifier( +SERVICES.append(reporters.MailNotifier( fromaddr="testbuild@taler.net", # notify from pass to fail, and viceversa. - generators=[mail_generator], + generators=[BuildStatusGenerator( + mode=('change','problem','failing','exception',), + builders=EMAIL_ALERT_BUILDERS, + message_formatter=reporters.MessageFormatter( + template_type='plain', + wantSteps=True, + wantLogs=True + ), +# add_logs=True + )], sendToInterestedUsers=False, extraRecipients=["buildfailures@taler.net"] -) +)) -buildslavetestNotifier = reporters.MailNotifier( - fromaddr="bb@taler.net", - generators=[test_generator], - sendToInterestedUsers=False, - useTls=True, - extraRecipients=['grothoff@taler.net'] -) -brokenLinksEmailNotifier = reporters.MailNotifier( - fromaddr="bb@taler.net", - generators=[docs_generator], - sendToInterestedUsers=False, - useTls=True, - extraRecipients=['linkcheck@taler.net'] -) +############# GENERAL PURPOSE SCHEDULERS ########################## -#sphinxErrorNotifier = reporters.MailNotifier( -# fromaddr="bb@taler.net", -# sendToInterestedUsers=False, -# addLogs=['build docs.stdio',], -# useTls=True, -# # notify if sphinx exits with error (command line option in Makefile turns warnings into exit 1) -# mode=('failing'), -# builders=('doc-builder',), -# extraRecipients=['sphinxerrors@taler.net'] -#) +# Workers that are done on wallet or deployment changes to master +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="taler-healthcheck-scheduler", + change_filter=util.ChangeFilter( + branch="master", + project_re="(wallet|deployment)" + ), + treeStableTimer=None, + builderNames=WALLETCHANGE_BUILDERS +)) -c["services"] = [ - IRC, - EMAIL, - brokenLinksEmailNotifier, -# sphinxErrorNotifier, - buildslavetestNotifier -] +SCHEDULERS.append(schedulers.SingleBranchScheduler( + name="all-scheduler", + change_filter=util.ChangeFilter( + branch_re="(master|stable)", + project_re="(backoffice|wallet-core|bank|exchange|" + "merchant|deployment|twister|sync|" + "help|taler-merchant-demos)" + ), + treeStableTimer=None, + builderNames=CODECHANGE_BUILDERS +)) + +# Scheduler for all nightly builds. +SCHEDULERS.append(schedulers.Nightly( + name="nightly-scheduler", + builderNames=list(NIGHTLY_BUILDERS), + branch="master", + hour=6, + minute=0 +)) -####### PROJECT IDENTITY +# Provide "force" button in the web UI. +SCHEDULERS.append(schedulers.ForceScheduler( + name="force-scheduler", + builderNames=list(BUILDER_LIST) +)) + + +####### Actual configuation initialization ############## + +# This is the dictionary that the buildmaster pays attention to. We also use +# a shorter alias to save typing. +c = BuildmasterConfig = {} + +# Silence warning and allow very basic phoning home. +c["buildbotNetUsageData"] = "basic" c["title"] = "GNU Taler" c["titleURL"] = "https://taler.net" +# the 'change_source' setting tells the buildmaster how it should +# find out about source code changes. +c["change_source"] = [changes.PBChangeSource(user="allcs", passwd="allcs")] + +# 'protocols' contains information about protocols which master +# will use for communicating with workers. You must define at +# least 'port' option that workers could connect to your master +# with this protocol. 'port' must match the value configured into +# the workers (with their --master option) +c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}} + +c["schedulers"] = SCHEDULERS + + # We use nginx to expose the BB under this URL. c["buildbotURL"] = "https://buildbot.taler.net/" +c["workers"] = WORKERS + +c["services"] = SERVICES + +c["builders"] = BUILDERS -BUILDER_LIST = [ - "buildmaster-builder", - "linkchecker-builder", - "buildslavetest-builder", - "doc-builder", - "lcov-builder", - "checker-builder", - "codespell-builder", - "auditor-builder", - "auditor-builder-test", - "auditor-builder-demo", - "demo-services-checker-builder", - "sites-builder", - "test-builder", - "wallet-builder", - "taler-demo-healthcheck-builder", - "taler-test-healthcheck-builder", -] # Authentication for Web UI. exec(compile(open('../taler_bb_userpass_db/taler_bb_userpass_db/db.py').read(), filename='db.py', mode='exec')) diff --git a/buildbot/master.cfg-GivingUpNewFormatting b/buildbot/master.cfg-GivingUpNewFormatting deleted file mode 100644 index 4663a0c..0000000 --- a/buildbot/master.cfg-GivingUpNewFormatting +++ /dev/null @@ -1,1159 +0,0 @@ -## -# This file is part of TALER -# (C) 2016-2020 Taler Systems SA -# -# TALER is free software; you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public -# License as published by the Free Software Foundation; either -# version 3, or (at your option) any later version. -# -# TALER is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty -# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with TALER; see the file COPYING. If not, -# see <http://www.gnu.org/licenses/> -# -# @author Florian Dold -# @author Marcello Stanisci -# @author ng0 -# @author Christian Grothoff -import re -from getpass import getuser -from buildbot.steps.source.git import Git -from buildbot.steps.shell import ShellCommand -from buildbot.plugins import (reporters, worker, - changes, schedulers, util) - -# This is a sample buildmaster config file. It must be -# installed as 'master.cfg' in your buildmaster's base -# directory. This is the dictionary that the buildmaster -# pays attention to. We also use a shorter alias to save -# typing. -c = BuildmasterConfig = {} - -# Silence warning and allow very basic phoning home. -c["buildbotNetUsageData"] = "basic" - - - -### Define Functions and Declarations - -# Convenience function that checks out a Git repository. -# First argument is the URL of the Git to clone, second -# the desired branch. Default is 'master'. -def git_step(repo,target_branch="master"): - return Git( - repourl=repo, - mode="full", - method="fresh", - logEnviron=False, - alwaysUseLatest=True, - haltOnFailure=True, - branch=target_branch - ) - -# Convenience function that runs 'make check' in a -# directory of the code inside of a netjail. -def jailed_check(package,srcdir): - return ShellCommand( - name="Test of " + srcdir + " in " + package, - description="Testing " + srcdir + " in " + package, - descriptionDone="Pass", - command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"], - workdir="../../sources/" + package + "/src/" + srcdir - ) - -# Convenience function that checks out the deployment. -def update_deployment(factory): - factory.addStep(ShellCommand( - name="clean old deployment", - description="cleaning previous deployment repository", - descriptionDone="Deployment removed", - command=["rm", "-rf", "deployment"], - workdir="../.." - )) - factory.addStep(ShellCommand( - name="update deployment", - description="fetching latest deployment repository", - descriptionDone="Deployment updated", - command=["git", "clone", "git://git.taler.net/deployment"], - workdir="../.." - )) - -# Create a FACTORY with a deployment.git checkout as the first step. -def create_factory_with_deployment(): - f = util.BuildFactory() - update_deployment (f); - return f - -########################################################################################### -# ------------- NEW FORMATTING STARTS HERE FOR NEW AND MODIFIED PROJECTS ----------------- - -# !!! This matters. Mis-ordering leads to failure. !!! -# Best Practices: Group by worker as: -# - comment for WORKER location (ex: "Find worker at: xyz shell account non taler.net") -# - FACTORY declaration -# - BUILDER declaration -# - SCHEDULER declaration -# -# NOTE: WORKER must be added to list: c["workers"] = [ -# NOTE: BUILDER must be added to list: c["builders"] = [ -# NOTE: BUILDER must be added to this list too: BUILDER_LIST = [ -########################################################################################### - -########################### -### --- buildslavetest --- - -# buildslavetest WORKER: buildslavetest@taler.net - -# buildslavetest FACTORY -BUILDSLAVETEST_FACTORY = create_factory_with_deployment() -BUILDSLAVETEST_FACTORY.addStep( - ShellCommand( - name="buildslavetest script (for testing purposes)", - description="Build Slave Test", - descriptionDone="buildslavetest: Done", - command=["./buildslavetest.sh"], - workdir="/home/buildslavetest/" - ) -) - -# buildslavetest BUILDER -BUILDSLAVETEST_BUILDER = util.BuilderConfig( - name="buildslavetest-builder", - workernames=["buildslavetest-worker"], - factory=BUILDSLAVETEST_FACTORY -) - -# buildslavetest SCHEDULER -# Run at 11am every day. -TEST_SCHEDULER = schedulers.Nightly( - name="test-scheduler", - builderNames=["buildslavetest-builder"], - branch="master", - hour=11, - minute=00 -) - -### /end ---buildslavetest - -####################### -### --- linkchecker --- - -# linkchecker WORKER Location: linkchecker@taler.net - -# linkchecker FACTORY -LINKCHECKER_FACTORY = create_factory_with_deployment() -LINKCHECKER_FACTORY.addStep( - ShellCommand( - name="linkchecker", - description="Check taler.net website for broken links && Notify", - descriptionDone="linkchecker links checked", - command=["./linkchecker.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -# linkchecker BUILDER -# Builder for linkchecker -LINKCHECKER_BUILDER = util.BuilderConfig( - name="linkchecker-builder", - workernames="linkchecker-worker", - factory=LINKCHECKER_FACTORY -) - -# linkchecker SCHEDULER -# Run at 7:30am every day. -LINKCHECKER_SCHEDULER = schedulers.Nightly( - name="linkchecker-scheduler", - builderNames="linkchecker-builder", - branch="master", - hour=7, - minute=30 -) - -#DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig( -# name="demo-services-checker-builder", -# workernames="demo-worker", -# factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY -#) - -### /end --- linkchecker --- - -# / ########################################################################################## - - -####### WORKERS - -# The 'workers' list defines the set of recognized workers. -# Each element is a Worker object, specifying a unique worker -# name and password. The same worker name and password must -# be configured on the worker. -c["workers"] = [ - - ## - # This worker restarts the buildmaster itself on - # changes to this file. - worker.Worker("buildmaster-worker", "buildmaster-pass"), - - ## - # This worker makes the code coverage and publishes it - # under the "lcov" Website. - worker.Worker("lcov-worker", "lcov-pass"), - - ## - # This worker builds everything and runs our 'make check' - # test suite against 'everything'. - worker.Worker("checker-worker", "checker-pass"), - - ## - # This worker builds manuals / API docs / tutorials. - # - worker.Worker("doc-worker", "doc-pass"), - - ## - # This worker builds Websites: www and stage. - # - worker.Worker("sites-worker", "sites-pass"), - - ## - # This worker builds Taler for the 'test' deployment. - worker.Worker("test-worker", "test-pass"), - - ## - # Tip reserve toppers. - worker.Worker("test-topper-worker", "test-topper-pass"), - worker.Worker("demo-topper-worker", "demo-topper-pass"), - - ## - # This worker compiles the auditor reports for the "green" - # demo deployment. - worker.Worker("test-auditor-worker", "test-auditor-pass"), - worker.Worker("demo-auditor-worker", "demo-auditor-pass"), - - ## - # This worker checks that all the services run under the - # 'demo' deployment are up&running. - worker.Worker("demo-worker", "demo-pass"), - - ## - # This worker builds wallet-core. - worker.Worker("wallet-worker", "wallet-pass"), - - ## - # health checks performed by wallet-cli for demo - worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass"), - - ## - # health checks performed by wallet-cli for test - worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass"), - - ## - # testing buildbot using the "buildslavetest" user (for no specific reason except it exists) - # Home: `buidlslavetest` @ taler.net - worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo"), - - ## - # linkchecker worker checks for dead links in the Website - # Home: linkchecker@taler.net - worker.Worker("linkchecker-worker", "linkchecker-worker-pass") -] - -# 'protocols' contains information about protocols which master -# will use for communicating with workers. You must define at -# least 'port' option that workers could connect to your master -# with this protocol. 'port' must match the value configured into -# the workers (with their --master option) -c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}} - -####### CHANGESOURCES - -# the 'change_source' setting tells the buildmaster how it should -# find out about source code changes. - -# NOTE: BB is bound to localhost -ALLCS = changes.PBChangeSource(user="allcs", passwd="allcs") - -c["change_source"] = [ALLCS] - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming -# changes. - -# Run at 6am every day: coverage analysis and auditor. -NIGHTLY_SCHEDULER = schedulers.Nightly( - name="nightly-scheduler", - builderNames=["lcov-builder", - "auditor-builder-demo", - "auditor-builder-test"], - branch="master", - hour=6, - minute=0 -) - -# Docs run if master or stable branch of 'docs' (or deployment) changed. -DOC_SCHEDULER = schedulers.SingleBranchScheduler( - name="periodic-doc-scheduler", - builderNames=["doc-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", project_re="(docs|deployment)" - ), - treeStableTimer=None -) - - -# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed: -def twister_web_page(change): - _change = change.asDict() - repo = _change.get("project") - if repo in ["www", "buywith", "deployment"]: - return True - - files = _change.get("files") - for file in files: - if re.search(r"web", file.get("name", "")) \ - and "twister" == repo: - return True - return False - -# Sites are re-build whenever deployment, www buywith, or twister changes. -SITES_SCHEDULER = schedulers.SingleBranchScheduler( - name="sites-scheduler", - builderNames=["sites-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", - filter_fn=twister_web_page - ), - treeStableTimer=None -) - -# Buildmaster is notified whenever deployment changes -BUILDMASTER_SCHEDULER = schedulers.SingleBranchScheduler( - name="buildmaster-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(deployment)" - ), - treeStableTimer=None, - builderNames=["buildmaster-builder"] -) - -# Wallet is re-build whenever wallet-core or deployment changes -WALLET_SCHEDULER = schedulers.SingleBranchScheduler( - name="wallet-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet-core|deployment)" - ), - treeStableTimer=None, - builderNames=["wallet-builder"] -) - -# Healthchecks are done on wallet or deployment changes to master -TALER_HEALTHCHECK_SCHEDULER = schedulers.SingleBranchScheduler( - name="taler-healthcheck-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet|deployment)" - ), - treeStableTimer=None, - builderNames=["taler-test-healthcheck-builder", - "taler-demo-healthcheck-builder"] -) - -# We check demo once per hour. -DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic( - name="demo-services-checker-scheduler", - periodicBuildTimer=60 * 60, # 1 hour - builderNames=["demo-services-checker-builder"] -) - -# We top-up the tipping reserve every 10 days. -TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic( - name="tip-reserve-topper-scheduler", - periodicBuildTimer=60 * 60 * 24 * 10, # 10 days - builderNames=["tip-reserve-topper-builder-demo", - "tip-reserve-topper-builder-test"] -) - -# Scheduler that triggers if anything changes -ALL_SCHEDULER = schedulers.SingleBranchScheduler( - name="all-scheduler", - change_filter=util.ChangeFilter( - branch_re="(master|stable)", - project_re="(backoffice|wallet-core|bank|exchange|" - "merchant|deployment|donations|twister|sync|" - "blog|help|survey|landing)" - ), - treeStableTimer=None, - builderNames=["test-builder", "checker-builder"] -) - -# Consider adding other Python parts, like the various frontends. -# NOTE: scheduler is NOT active! (commented out below) -LINT_SCHEDULER = schedulers.SingleBranchScheduler( - name="lint-scheduler", - change_filter=util.ChangeFilter( - branch="master", project_re="(bank|donations|survey|blog)" - ), - treeStableTimer=None, - builderNames=["lint-builder"] -) - -# Provide "force" button in the web UI. -FORCE_SCHEDULER = schedulers.ForceScheduler( - name="force-scheduler", - builderNames=[ - "lcov-builder", - "checker-builder", - "auditor-builder-test", - "auditor-builder-demo", - "test-builder", - "doc-builder", - "linkchecker-builder", - "buildslavetest-builder", - "sites-builder", - "wallet-builder", - "taler-test-healthcheck-builder", - "taler-demo-healthcheck-builder", - "tip-reserve-topper-builder-test", - "tip-reserve-topper-builder-demo" - ] -) - -c["schedulers"] = [ - TEST_SCHEDULER, - NIGHTLY_SCHEDULER, - TIP_RESERVE_TOPPER_SCHEDULER, - DEMO_SERVICES_CHECKER_SCHEDULER, - DOC_SCHEDULER, - BUILDMASTER_SCHEDULER, - SITES_SCHEDULER, - ALL_SCHEDULER, - FORCE_SCHEDULER, - WALLET_SCHEDULER, - TALER_HEALTHCHECK_SCHEDULER, - - ## - # Rarely/never used, excluding. - # DEBUG_SCHEDULER, - # LINT_SCHEDULER -] - -####### BUILDERS - -# The 'builders' list defines the Builders, which tell Buildbot -# how to perform a build: what steps, and which workers can execute -# them. Note that any particular build will only take place on -# one worker. - - - - - -# Factory to update the buildmaster itself. -BUILDMASTER_FACTORY = create_factory_with_deployment() -BUILDMASTER_FACTORY.addStep( - ShellCommand( - name="restart buildmaster", - description="trigger buildmaster restart with new configuration", - descriptionDone="Buildmaster updated", - command=["buildbot", "reconfig"], - workdir="../../master/" - ) -) - -TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) - -TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) - - -WALLET_FACTORY = create_factory_with_deployment() -WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -WALLET_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test", - description="Running wallet tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose"], - workdir="build/", - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) -# WALLET_FACTORY.addStep( -# ShellCommand( -# name="lint", -# description="Linting the wallet", -# descriptionDone="Linting done", -# command=["make", "lint"], -# workdir="build/" -# ) -# ) - -DEBUG_FACTORY = util.BuildFactory() -DEBUG_FACTORY.addStep( - ShellCommand( - name="echo debug", - description="just echoing a word", - descriptionDone="builder responded", - command=["echo", "I'm here!"] - ) -) - - -def lint_dispatcher(project): - return "./lint_%s.sh" % project - - -LINT_FACTORY = util.BuildFactory() -LINT_FACTORY.addStep( - ShellCommand( - name="Python linter", - description="linting Python", - descriptionDone="linting done", - command=util.Transform(lint_dispatcher, util.Property("project")), - workdir="../../deployment/taler-build" - ) -) - - -INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="remove old binaries", - description="removing binaries from previous runs", - descriptionDone="Setup clean", - command=["rm", "-rf", "local"], - workdir="../.." - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch gnunet.org sources", - description="fetching latest deployment repositories from git.gnunet.org", - descriptionDone="GNUnet code base updated", - command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"], - workdir="../../deployment/buildbot" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch taler.net sources", - description="fetching latest deployment repositories from git.taler.net", - descriptionDone="Taler code base updated", - command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "anastasis", "bank", "twister"], - workdir="../../deployment/buildbot" - ) -) -INTEGRATIONTEST_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install of wallet", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running local bootstrap of wallet", - descriptionDone="wallet bootstrap done", - command=["bootstrap"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running local configure of wallet", - descriptionDone="wallet configure done", - command=["configure", "--prefix=$HOME/local/"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running local make of wallet", - descriptionDone="wallet make done", - command=["make"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=False - ) -) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "microhttpd")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "testcurl")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "testzzuf")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "util")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "pq")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "curl")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "json")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("twister", "test")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "util")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "curl")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "mhd")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "pq")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "json")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "bank-lib")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchangedb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "auditordb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchange")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "auditor")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "lib")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchange-tools")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "testing")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "benchmark")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "backenddb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "backend")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "lib")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "merchant-tools")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "util")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "syncdb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "sync")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "lib")) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="bank check", - description="Testing Taler Python bank", - descriptionDone="Done", - command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"], - workdir="../../sources/bank", - haltOnFailure=False, - env={'PYTHONUSERBASE': "$HOME/local" } - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="wallet check", - description="Testing wallet-core", - descriptionDone="Done", - command=["make", "check"], - workdir="../../sources/wallet-core", - haltOnFailure=False - ) -) - - -# FIXME: INTEGRATIONTEST_FACTORY is incomplete at this time: we still need -# to do the actual 'make check' steps (in a jail!) - - -LCOV_FACTORY = create_factory_with_deployment() -LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -LCOV_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install of wallet", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -LCOV_FACTORY.addStep( - ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - env={'TALER_COVERAGE': "1"} - ) -) -LCOV_FACTORY.addStep( - ShellCommand( - name="coverage generation", - description="running tests", - descriptionDone="generating HTML report", - command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "./coverage.sh"], - workdir="../../deployment/buildbot", - ) -) - -AUDITOR_FACTORY_TEST = create_factory_with_deployment() -AUDITOR_FACTORY_TEST.addStep( - ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/taler-test/"} - ) -) - -AUDITOR_FACTORY_DEMO = create_factory_with_deployment() -AUDITOR_FACTORY_DEMO.addStep( - ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/demo/active-home/"} - ) -) - -TIP_RESERVE_TOPPER_FACTORY = create_factory_with_deployment() -TIP_RESERVE_TOPPER_FACTORY.addStep( - ShellCommand( - name="tip reserve topper", - description="Topping the tip reserve.", - descriptionDone="Tip reserve has been topped.", - command=["./top_reserve.sh"], - workdir="../../deployment/buildbot" - ) -) - -BUILD_FACTORY = create_factory_with_deployment() -BUILD_FACTORY.addStep( - ShellCommand( - name="build", - description="Building all Taler codebase.", - descriptionDone="Taler built.", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="config", - description="Generating configuration file.", - descriptionDone="Configuration file generated.", - command=["./config.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="keys generation and sign", - description="Generating exchange keys, and auditor-sign them.", - descriptionDone="Exchange keys generated, and auditor-signed.", - command=["./keys.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'BRANCH': util.Property("branch")} - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="restart services", - description="Restarting inactive blue-green party.", - descriptionDone="Restarting Taler.", - command=["./restart.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'BRANCH': util.Property("branch")} - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="check services correctly restarted", - description="Checking services are correctly restarted.", - descriptionDone="All services are correctly restarted.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'DEPLOYMENT': "test"} - ) -) - -SELENIUM_FACTORY = create_factory_with_deployment() -SELENIUM_FACTORY.addStep( - ShellCommand( - name="selenium", - description="Headless browser test", - descriptionDone="Test finished", - command=["launch_selenium_test"], - env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"} - ) -) - -DOC_FACTORY = create_factory_with_deployment() -DOC_FACTORY.addStep( - ShellCommand( - name="build docs", - description="Building documentation", - descriptionDone="Documentation built.", - command=["./build-docs.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -SITES_FACTORY = create_factory_with_deployment() -SITES_FACTORY.addStep( - ShellCommand( - name="build Web sites", - description="Building all the Taler homepages", - descriptionDone="Sites built.", - command=["./build-sites.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="demo services checker", - description="Checking demo services are online", - descriptionDone="Demo services are online!.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - # Needed to test the 'demo' deployment. - env={"DEPLOYMENT": "demo"} - ) -) - - -##### BUILDERS DEFINED - -# Builder to update the buildmaster itself. -BUILDMASTER_BUILDER = util.BuilderConfig( - name="buildmaster-builder", - workernames=["buildmaster-worker"], - factory=BUILDMASTER_FACTORY -) - -# This builder is NOT ACTIVE! -LINT_BUILDER = util.BuilderConfig( - name="lint-builder", - workernames=["lint-worker"], - factory=LINT_FACTORY -) - -LCOV_BUILDER = util.BuilderConfig( - name="lcov-builder", - workernames=["lcov-worker"], - factory=LCOV_FACTORY -) - -CHECKER_BUILDER = util.BuilderConfig( - name="checker-builder", - workernames=["checker-worker"], - factory=INTEGRATIONTEST_FACTORY -) - -AUDITOR_BUILDER_TEST = util.BuilderConfig( - name="auditor-builder-test", - workernames=["test-auditor-worker"], - factory=AUDITOR_FACTORY_TEST -) - -AUDITOR_BUILDER_DEMO = util.BuilderConfig( - name="auditor-builder-demo", - workernames=["demo-auditor-worker"], - factory=AUDITOR_FACTORY_DEMO -) - -TIP_RESERVE_TOPPER_BUILDER_DEMO = util.BuilderConfig( - name="tip-reserve-topper-builder-demo", - workernames=["demo-topper-worker"], - factory=TIP_RESERVE_TOPPER_FACTORY -) - -TIP_RESERVE_TOPPER_BUILDER_TEST = util.BuilderConfig( - name="tip-reserve-topper-builder-test", - workernames=["test-topper-worker"], - factory=TIP_RESERVE_TOPPER_FACTORY -) - -DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig( - name="demo-services-checker-builder", - workernames="demo-worker", - factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY -) - -TEST_BUILDER = util.BuilderConfig( - name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY -) - -SELENIUM_BUILDER = util.BuilderConfig( - name="selenium-builder", - workernames=["selenium-worker"], - factory=SELENIUM_FACTORY -) - -DOC_BUILDER = util.BuilderConfig( - name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY -) - -SITES_BUILDER = util.BuilderConfig( - name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY -) - -WALLET_BUILDER = util.BuilderConfig( - name="wallet-builder", - workernames=["wallet-worker"], - factory=WALLET_FACTORY -) - -TALER_TEST_HEALTHCHECK_BUILDER = util.BuilderConfig( - name="taler-test-healthcheck-builder", - workernames=["taler-test-healthcheck"], - factory=TALER_TEST_HEALTHCHECK_FACTORY -) - -TALER_DEMO_HEALTHCHECK_BUILDER = util.BuilderConfig( - name="taler-demo-healthcheck-builder", - workernames=["taler-demo-healthcheck"], - factory=TALER_DEMO_HEALTHCHECK_FACTORY -) - -c["builders"] = [ - LINKCHECKER_BUILDER, - BUILDSLAVETEST_BUILDER, - BUILDMASTER_BUILDER, - LCOV_BUILDER, - CHECKER_BUILDER, - AUDITOR_BUILDER_TEST, - AUDITOR_BUILDER_DEMO, - TEST_BUILDER, - TIP_RESERVE_TOPPER_BUILDER_DEMO, - TIP_RESERVE_TOPPER_BUILDER_TEST, - DEMO_SERVICES_CHECKER_BUILDER, - DOC_BUILDER, - SITES_BUILDER, - WALLET_BUILDER, - TALER_DEMO_HEALTHCHECK_BUILDER, - TALER_TEST_HEALTHCHECK_BUILDER, -] - -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter -# targets. The status of each build will be pushed to these -# targets. buildbot/reporters/*.py has a variety to choose from, -# like IRC bots. - -IRC = reporters.IRC( - "irc.eu.freenode.net", - "taler-bb", - useColors=False, - channels=[{ - "channel": "#taler" - }], - password="taler-bb-pass19", - notify_events={ - 'exception': 1, - 'successToFailure': 1, - 'failureToSuccess': 1 - } -) - -EMAIL = reporters.MailNotifier( - fromaddr="testbuild@taler.net", - sendToInterestedUsers=False, - # notify from pass to fail, and viceversa. - mode=("change"), - builders=( - "lcov-builder", - "checker-builder", - "doc-builder", - "test-builder", - "wallet-builder", - ## - # Rarely/never used, excluding. - # "selenium-builder" - ), - extraRecipients=["buildfailures@taler.net"], - subject="Taler build." -) - -c["services"] = [IRC, EMAIL] - -####### PROJECT IDENTITY - -c["title"] = "GNU Taler" -c["titleURL"] = "https://taler.net" - -# We use nginx to expose the BB under this URL. -c["buildbotURL"] = "https://buildbot.taler.net/" - - -BUILDER_LIST = [ - "linkchecker-builder", - "buildslavetest-builder", - "buildmaster-builder", - "doc-builder", - "lcov-builder", - "checker-builder", - "auditor-builder", - "auditor-builder-test", - "auditor-builder-demo", - "demo-services-checker-builder", - "tip-reserve-topper-builder-demo", - "tip-reserve-topper-builder-test", - "sites-builder", - "test-builder", - "wallet-builder", - "taler-demo-healthcheck-builder", - "taler-test-healthcheck-builder", -] - -# Authentication for Web UI. -exec(compile(open('../taler_bb_userpass_db/taler_bb_userpass_db/db.py').read(), filename='db.py', mode='exec')) - -authz = util.Authz( - allowRules=[ - util.ForceBuildEndpointMatcher(role="admins", - builder=b) - for b in BUILDER_LIST - ] + [ - util.StopBuildEndpointMatcher(role="admins", - builder=b) - for b in BUILDER_LIST - ] + [ - util.RebuildBuildEndpointMatcher(role="admins", - builder=b) - for b in BUILDER_LIST - ] + [ - util.ForceBuildEndpointMatcher(role="norole", - builder=b) - for b in BUILDER_LIST - ] + [ - util.StopBuildEndpointMatcher(role="norole", - builder=b) - for b in BUILDER_LIST - ] + [ - util.RebuildBuildEndpointMatcher(role="norole", - builder=b) - for b in BUILDER_LIST - ], - roleMatchers=[ - util.RolesFromUsername( - roles=["admins"], - usernames=["marcello", "florian", "christian", "buckE"] - ) - ] -) - -# minimalistic config to activate new web UI -# -- formerly commented out as not packaged properly in Debian and others, see -# https://bugzilla.redhat.com/show_bug.cgi?id=1557687 -# -- un-commented as "pip install buildbot.www" performed from buildbot-master account -c["www"] = { - "port": 8010, - "plugins": { - "waterfall_view": {}, - "console_view": {} - }, - "allowed_origins": ["https://*.taler.net"], - "avatar_methods": [], - "auth": util.UserPasswordAuth(USER_PASSWORD_DB), - "authz": authz -} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its - # state. You can leave this at its default for all but the - # largest installations. - "db_url": "sqlite:///state.sqlite", -} diff --git a/buildbot/master.cfg.BACK-movingFunctionsAround b/buildbot/master.cfg.BACK-movingFunctionsAround deleted file mode 100644 index c40a6c5..0000000 --- a/buildbot/master.cfg.BACK-movingFunctionsAround +++ /dev/null @@ -1,1153 +0,0 @@ -## -# This file is part of TALER -# (C) 2016-2020 Taler Systems SA -# -# TALER is free software; you can redistribute it and/or -# modify it under the terms of the GNU Affero General Public -# License as published by the Free Software Foundation; either -# version 3, or (at your option) any later version. -# -# TALER is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty -# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public -# License along with TALER; see the file COPYING. If not, -# see <http://www.gnu.org/licenses/> -# -# @author Florian Dold -# @author Marcello Stanisci -# @author ng0 -# @author Christian Grothoff -import re -from getpass import getuser -from buildbot.steps.source.git import Git -from buildbot.steps.shell import ShellCommand -from buildbot.plugins import (reporters, worker, - changes, schedulers, util) - -# This is a sample buildmaster config file. It must be -# installed as 'master.cfg' in your buildmaster's base -# directory. This is the dictionary that the buildmaster -# pays attention to. We also use a shorter alias to save -# typing. -c = BuildmasterConfig = {} - -# Silence warning and allow very basic phoning home. -c["buildbotNetUsageData"] = "basic" - -########################################################################################### -# ------------- NEW FORMATTING STARTS HERE FOR NEW AND MODIFIED PROJECTS ----------------- - -# !!! This matters. Mis-ordering leads to failure. !!! -# Best Practices: Group by worker as: -# - comment for worker location (ex: "Find worker at: xyz shell account non taler.net") -# - SCHEDULER -# - FACTORY function -# - BUILDER function -# -# NOTE: WORKER must be added to list: c["workers"] = [ -# NOTE: BUILDER must be added to list: c["builders"] = [ -# NOTE: BUILDER must be added to this list too: BUILDER_LIST = [ -########################################################################################### - -########################### -### --- buildslavetest --- - -# buildslavetest WORKER: buildslavetest@taler.net - -# buildslavetest SCHEDULER -# Run at 11am every day. -TEST_SCHEDULER = schedulers.Nightly( - name="test-scheduler", - builderNames=["buildslavetest-builder"], - branch="master", - hour=11, - minute=00 -) - -# buildslavetest FACTORY -BUILDSLAVETEST_FACTORY = create_factory_with_deployment() -BUILDSLAVETEST_FACTORY.addStep( - ShellCommand( - name="buildslavetest script (for testing purposes)", - description="Build Slave Test", - descriptionDone="buildslavetest: Done", - command=["./buildslavetest.sh"], - workdir="/home/buildslavetest/" - ) -) - -# buildslavetest BUILDER -BUILDSLAVETEST_BUILDER = util.BuilderConfig( - name="buildslavetest-builder", - workernames=["buildslavetest-worker"], - factory=BUILDSLAVETEST_FACTORY -) - -### /end ---buildslavetest - -####################### -### --- linkchecker --- - -# linkchecker WORKER Location: linkchecker@taler.net - -# linkchecker SCHEDULER -# Run at 7:30am every day. -LINKCHECKER_SCHEDULER = schedulers.Nightly( - name="linkchecker-scheduler", - builderNames="linkchecker-builder", - branch="master", - hour=7, - minute=30 -) - -# linkchecker BUILDER -# Builder for linkchecker -LINKCHECKER_BUILDER = util.BuilderConfig( - name="linkchecker-builder", - workernames="linkchecker-worker", - factory=LINKCHECKER_FACTORY -) - -DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig( - name="demo-services-checker-builder", - workernames="demo-worker", - factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY -) - -# linkchecker FACTORY -LINKCHECKER_FACTORY = create_factory_with_deployment() -LINKCHECKER_FACTORY.addStep( - ShellCommand( - name="linkchecker", - description="Check taler.net website for broken links && Notify", - descriptionDone="linkchecker links checked", - command=["./linkchecker.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -### /end --- linkchecker --- - -# / ########################################################################################## - - -####### WORKERS - -# The 'workers' list defines the set of recognized workers. -# Each element is a Worker object, specifying a unique worker -# name and password. The same worker name and password must -# be configured on the worker. -c["workers"] = [ - - ## - # This worker restarts the buildmaster itself on - # changes to this file. - worker.Worker("buildmaster-worker", "buildmaster-pass"), - - ## - # This worker makes the code coverage and publishes it - # under the "lcov" Website. - worker.Worker("lcov-worker", "lcov-pass"), - - ## - # This worker builds everything and runs our 'make check' - # test suite against 'everything'. - worker.Worker("checker-worker", "checker-pass"), - - ## - # This worker builds manuals / API docs / tutorials. - # - worker.Worker("doc-worker", "doc-pass"), - - ## - # This worker builds Websites: www and stage. - # - worker.Worker("sites-worker", "sites-pass"), - - ## - # This worker builds Taler for the 'test' deployment. - worker.Worker("test-worker", "test-pass"), - - ## - # Tip reserve toppers. - worker.Worker("test-topper-worker", "test-topper-pass"), - worker.Worker("demo-topper-worker", "demo-topper-pass"), - - ## - # This worker compiles the auditor reports for the "green" - # demo deployment. - worker.Worker("test-auditor-worker", "test-auditor-pass"), - worker.Worker("demo-auditor-worker", "demo-auditor-pass"), - - ## - # This worker checks that all the services run under the - # 'demo' deployment are up&running. - worker.Worker("demo-worker", "demo-pass"), - - ## - # This worker builds wallet-core. - worker.Worker("wallet-worker", "wallet-pass"), - - ## - # health checks performed by wallet-cli for demo - worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass"), - - ## - # health checks performed by wallet-cli for test - worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass"), - - ## - # testing buildbot using the "buildslavetest" user (for no specific reason except it exists) - # Home: `buidlslavetest` @ taler.net - worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo"), - - ## - # linkchecker worker checks for dead links in the Website - # Home: linkchecker@taler.net - worker.Worker("linkchecker-worker", "linkchecker-worker-pass") -] - -# 'protocols' contains information about protocols which master -# will use for communicating with workers. You must define at -# least 'port' option that workers could connect to your master -# with this protocol. 'port' must match the value configured into -# the workers (with their --master option) -c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}} - -####### CHANGESOURCES - -# the 'change_source' setting tells the buildmaster how it should -# find out about source code changes. - -# NOTE: BB is bound to localhost -ALLCS = changes.PBChangeSource(user="allcs", passwd="allcs") - -c["change_source"] = [ALLCS] - -####### SCHEDULERS - -# Configure the Schedulers, which decide how to react to incoming -# changes. - -# Run at 6am every day: coverage analysis and auditor. -NIGHTLY_SCHEDULER = schedulers.Nightly( - name="nightly-scheduler", - builderNames=["lcov-builder", - "auditor-builder-demo", - "auditor-builder-test"], - branch="master", - hour=6, - minute=0 -) - -# Docs run if master or stable branch of 'docs' (or deployment) changed. -DOC_SCHEDULER = schedulers.SingleBranchScheduler( - name="periodic-doc-scheduler", - builderNames=["doc-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", project_re="(docs|deployment)" - ), - treeStableTimer=None -) - - -# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed: -def twister_web_page(change): - _change = change.asDict() - repo = _change.get("project") - if repo in ["www", "buywith", "deployment"]: - return True - - files = _change.get("files") - for file in files: - if re.search(r"web", file.get("name", "")) \ - and "twister" == repo: - return True - return False - -# Sites are re-build whenever deployment, www buywith, or twister changes. -SITES_SCHEDULER = schedulers.SingleBranchScheduler( - name="sites-scheduler", - builderNames=["sites-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", - filter_fn=twister_web_page - ), - treeStableTimer=None -) - -# Buildmaster is notified whenever deployment changes -BUILDMASTER_SCHEDULER = schedulers.SingleBranchScheduler( - name="buildmaster-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(deployment)" - ), - treeStableTimer=None, - builderNames=["buildmaster-builder"] -) - -# Wallet is re-build whenever wallet-core or deployment changes -WALLET_SCHEDULER = schedulers.SingleBranchScheduler( - name="wallet-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet-core|deployment)" - ), - treeStableTimer=None, - builderNames=["wallet-builder"] -) - -# Healthchecks are done on wallet or deployment changes to master -TALER_HEALTHCHECK_SCHEDULER = schedulers.SingleBranchScheduler( - name="taler-healthcheck-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet|deployment)" - ), - treeStableTimer=None, - builderNames=["taler-test-healthcheck-builder", - "taler-demo-healthcheck-builder"] -) - -# We check demo once per hour. -DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic( - name="demo-services-checker-scheduler", - periodicBuildTimer=60 * 60, # 1 hour - builderNames=["demo-services-checker-builder"] -) - -# We top-up the tipping reserve every 10 days. -TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic( - name="tip-reserve-topper-scheduler", - periodicBuildTimer=60 * 60 * 24 * 10, # 10 days - builderNames=["tip-reserve-topper-builder-demo", - "tip-reserve-topper-builder-test"] -) - -# Scheduler that triggers if anything changes -ALL_SCHEDULER = schedulers.SingleBranchScheduler( - name="all-scheduler", - change_filter=util.ChangeFilter( - branch_re="(master|stable)", - project_re="(backoffice|wallet-core|bank|exchange|" - "merchant|deployment|donations|twister|sync|" - "blog|help|survey|landing)" - ), - treeStableTimer=None, - builderNames=["test-builder", "checker-builder"] -) - -# Consider adding other Python parts, like the various frontends. -# NOTE: scheduler is NOT active! (commented out below) -LINT_SCHEDULER = schedulers.SingleBranchScheduler( - name="lint-scheduler", - change_filter=util.ChangeFilter( - branch="master", project_re="(bank|donations|survey|blog)" - ), - treeStableTimer=None, - builderNames=["lint-builder"] -) - -# Provide "force" button in the web UI. -FORCE_SCHEDULER = schedulers.ForceScheduler( - name="force-scheduler", - builderNames=[ - "lcov-builder", - "checker-builder", - "auditor-builder-test", - "auditor-builder-demo", - "test-builder", - "doc-builder", - "linkchecker-builder", - "buildslavetest-builder", - "sites-builder", - "wallet-builder", - "taler-test-healthcheck-builder", - "taler-demo-healthcheck-builder", - "tip-reserve-topper-builder-test", - "tip-reserve-topper-builder-demo" - ] -) - -c["schedulers"] = [ - TEST_SCHEDULER, - NIGHTLY_SCHEDULER, - TIP_RESERVE_TOPPER_SCHEDULER, - DEMO_SERVICES_CHECKER_SCHEDULER, - DOC_SCHEDULER, - BUILDMASTER_SCHEDULER, - SITES_SCHEDULER, - ALL_SCHEDULER, - FORCE_SCHEDULER, - WALLET_SCHEDULER, - TALER_HEALTHCHECK_SCHEDULER, - - ## - # Rarely/never used, excluding. - # DEBUG_SCHEDULER, - # LINT_SCHEDULER -] - -####### BUILDERS - -# The 'builders' list defines the Builders, which tell Buildbot -# how to perform a build: what steps, and which workers can execute -# them. Note that any particular build will only take place on -# one worker. - - -# Convenience function that checks out a Git repository. -# First argument is the URL of the Git to clone, second -# the desired branch. Default is 'master'. -def git_step(repo,target_branch="master"): - return Git( - repourl=repo, - mode="full", - method="fresh", - logEnviron=False, - alwaysUseLatest=True, - haltOnFailure=True, - branch=target_branch - ) - - -# Convenience function that runs 'make check' in a -# directory of the code inside of a netjail. -def jailed_check(package,srcdir): - return ShellCommand( - name="Test of " + srcdir + " in " + package, - description="Testing " + srcdir + " in " + package, - descriptionDone="Pass", - command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"], - workdir="../../sources/" + package + "/src/" + srcdir - ) - -# Convenience function that checks out the deployment. -def update_deployment(factory): - factory.addStep(ShellCommand( - name="clean old deployment", - description="cleaning previous deployment repository", - descriptionDone="Deployment removed", - command=["rm", "-rf", "deployment"], - workdir="../.." - )) - factory.addStep(ShellCommand( - name="update deployment", - description="fetching latest deployment repository", - descriptionDone="Deployment updated", - command=["git", "clone", "git://git.taler.net/deployment"], - workdir="../.." - )) - -# Create a FACTORY with a deployment.git checkout as the first step. -def create_factory_with_deployment(): - f = util.BuildFactory() - update_deployment (f); - return f - -# Factory to update the buildmaster itself. -BUILDMASTER_FACTORY = create_factory_with_deployment() -BUILDMASTER_FACTORY.addStep( - ShellCommand( - name="restart buildmaster", - description="trigger buildmaster restart with new configuration", - descriptionDone="Buildmaster updated", - command=["buildbot", "reconfig"], - workdir="../../master/" - ) -) - -TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) - -TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) - - -WALLET_FACTORY = create_factory_with_deployment() -WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -WALLET_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test", - description="Running wallet tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose"], - workdir="build/", - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) -# WALLET_FACTORY.addStep( -# ShellCommand( -# name="lint", -# description="Linting the wallet", -# descriptionDone="Linting done", -# command=["make", "lint"], -# workdir="build/" -# ) -# ) - -DEBUG_FACTORY = util.BuildFactory() -DEBUG_FACTORY.addStep( - ShellCommand( - name="echo debug", - description="just echoing a word", - descriptionDone="builder responded", - command=["echo", "I'm here!"] - ) -) - - -def lint_dispatcher(project): - return "./lint_%s.sh" % project - - -LINT_FACTORY = util.BuildFactory() -LINT_FACTORY.addStep( - ShellCommand( - name="Python linter", - description="linting Python", - descriptionDone="linting done", - command=util.Transform(lint_dispatcher, util.Property("project")), - workdir="../../deployment/taler-build" - ) -) - - -INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="remove old binaries", - description="removing binaries from previous runs", - descriptionDone="Setup clean", - command=["rm", "-rf", "local"], - workdir="../.." - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch gnunet.org sources", - description="fetching latest deployment repositories from git.gnunet.org", - descriptionDone="GNUnet code base updated", - command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"], - workdir="../../deployment/buildbot" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch taler.net sources", - description="fetching latest deployment repositories from git.taler.net", - descriptionDone="Taler code base updated", - command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "anastasis", "bank", "twister"], - workdir="../../deployment/buildbot" - ) -) -INTEGRATIONTEST_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install of wallet", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running local bootstrap of wallet", - descriptionDone="wallet bootstrap done", - command=["bootstrap"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running local configure of wallet", - descriptionDone="wallet configure done", - command=["configure", "--prefix=$HOME/local/"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running local make of wallet", - descriptionDone="wallet make done", - command=["make"], - workdir="../../sources/wallet-core" - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=False - ) -) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "microhttpd")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "testcurl")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", "testzzuf")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "util")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "pq")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "curl")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", "json")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("twister", "test")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "util")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "curl")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "mhd")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "pq")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "json")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "bank-lib")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchangedb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "auditordb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchange")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "auditor")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "lib")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "exchange-tools")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "testing")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", "benchmark")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "backenddb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "backend")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "lib")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", "merchant-tools")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "util")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "syncdb")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "sync")) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", "lib")) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="bank check", - description="Testing Taler Python bank", - descriptionDone="Done", - command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"], - workdir="../../sources/bank", - haltOnFailure=False, - env={'PYTHONUSERBASE': "$HOME/local" } - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="wallet check", - description="Testing wallet-core", - descriptionDone="Done", - command=["make", "check"], - workdir="../../sources/wallet-core", - haltOnFailure=False - ) -) - - -# FIXME: INTEGRATIONTEST_FACTORY is incomplete at this time: we still need -# to do the actual 'make check' steps (in a jail!) - - -LCOV_FACTORY = create_factory_with_deployment() -LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -LCOV_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install of wallet", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"], - workdir="build/" - ) -) -LCOV_FACTORY.addStep( - ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - env={'TALER_COVERAGE': "1"} - ) -) -LCOV_FACTORY.addStep( - ShellCommand( - name="coverage generation", - description="running tests", - descriptionDone="generating HTML report", - command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "./coverage.sh"], - workdir="../../deployment/buildbot", - ) -) - -AUDITOR_FACTORY_TEST = create_factory_with_deployment() -AUDITOR_FACTORY_TEST.addStep( - ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/taler-test/"} - ) -) - -AUDITOR_FACTORY_DEMO = create_factory_with_deployment() -AUDITOR_FACTORY_DEMO.addStep( - ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/demo/active-home/"} - ) -) - -TIP_RESERVE_TOPPER_FACTORY = create_factory_with_deployment() -TIP_RESERVE_TOPPER_FACTORY.addStep( - ShellCommand( - name="tip reserve topper", - description="Topping the tip reserve.", - descriptionDone="Tip reserve has been topped.", - command=["./top_reserve.sh"], - workdir="../../deployment/buildbot" - ) -) - -BUILD_FACTORY = create_factory_with_deployment() -BUILD_FACTORY.addStep( - ShellCommand( - name="build", - description="Building all Taler codebase.", - descriptionDone="Taler built.", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="config", - description="Generating configuration file.", - descriptionDone="Configuration file generated.", - command=["./config.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="keys generation and sign", - description="Generating exchange keys, and auditor-sign them.", - descriptionDone="Exchange keys generated, and auditor-signed.", - command=["./keys.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'BRANCH': util.Property("branch")} - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="restart services", - description="Restarting inactive blue-green party.", - descriptionDone="Restarting Taler.", - command=["./restart.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'BRANCH': util.Property("branch")} - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="check services correctly restarted", - description="Checking services are correctly restarted.", - descriptionDone="All services are correctly restarted.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'DEPLOYMENT': "test"} - ) -) - -SELENIUM_FACTORY = create_factory_with_deployment() -SELENIUM_FACTORY.addStep( - ShellCommand( - name="selenium", - description="Headless browser test", - descriptionDone="Test finished", - command=["launch_selenium_test"], - env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"} - ) -) - -DOC_FACTORY = create_factory_with_deployment() -DOC_FACTORY.addStep( - ShellCommand( - name="build docs", - description="Building documentation", - descriptionDone="Documentation built.", - command=["./build-docs.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -SITES_FACTORY = create_factory_with_deployment() -SITES_FACTORY.addStep( - ShellCommand( - name="build Web sites", - description="Building all the Taler homepages", - descriptionDone="Sites built.", - command=["./build-sites.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) - -DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="demo services checker", - description="Checking demo services are online", - descriptionDone="Demo services are online!.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - # Needed to test the 'demo' deployment. - env={"DEPLOYMENT": "demo"} - ) -) - - -##### BUILDERS DEFINED - -# Builder to update the buildmaster itself. -BUILDMASTER_BUILDER = util.BuilderConfig( - name="buildmaster-builder", - workernames=["buildmaster-worker"], - factory=BUILDMASTER_FACTORY -) - -# This builder is NOT ACTIVE! -LINT_BUILDER = util.BuilderConfig( - name="lint-builder", - workernames=["lint-worker"], - factory=LINT_FACTORY -) - -LCOV_BUILDER = util.BuilderConfig( - name="lcov-builder", - workernames=["lcov-worker"], - factory=LCOV_FACTORY -) - -CHECKER_BUILDER = util.BuilderConfig( - name="checker-builder", - workernames=["checker-worker"], - factory=INTEGRATIONTEST_FACTORY -) - -AUDITOR_BUILDER_TEST = util.BuilderConfig( - name="auditor-builder-test", - workernames=["test-auditor-worker"], - factory=AUDITOR_FACTORY_TEST -) - -AUDITOR_BUILDER_DEMO = util.BuilderConfig( - name="auditor-builder-demo", - workernames=["demo-auditor-worker"], - factory=AUDITOR_FACTORY_DEMO -) - -TIP_RESERVE_TOPPER_BUILDER_DEMO = util.BuilderConfig( - name="tip-reserve-topper-builder-demo", - workernames=["demo-topper-worker"], - factory=TIP_RESERVE_TOPPER_FACTORY -) - -TIP_RESERVE_TOPPER_BUILDER_TEST = util.BuilderConfig( - name="tip-reserve-topper-builder-test", - workernames=["test-topper-worker"], - factory=TIP_RESERVE_TOPPER_FACTORY -) - -DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig( - name="demo-services-checker-builder", - workernames="demo-worker", - factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY -) - -TEST_BUILDER = util.BuilderConfig( - name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY -) - -SELENIUM_BUILDER = util.BuilderConfig( - name="selenium-builder", - workernames=["selenium-worker"], - factory=SELENIUM_FACTORY -) - -DOC_BUILDER = util.BuilderConfig( - name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY -) - -SITES_BUILDER = util.BuilderConfig( - name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY -) - -WALLET_BUILDER = util.BuilderConfig( - name="wallet-builder", - workernames=["wallet-worker"], - factory=WALLET_FACTORY -) - -TALER_TEST_HEALTHCHECK_BUILDER = util.BuilderConfig( - name="taler-test-healthcheck-builder", - workernames=["taler-test-healthcheck"], - factory=TALER_TEST_HEALTHCHECK_FACTORY -) - -TALER_DEMO_HEALTHCHECK_BUILDER = util.BuilderConfig( - name="taler-demo-healthcheck-builder", - workernames=["taler-demo-healthcheck"], - factory=TALER_DEMO_HEALTHCHECK_FACTORY -) - -c["builders"] = [ - LINKCHECKER_BUILDER, - BUILDSLAVETEST_BUILDER, - BUILDMASTER_BUILDER, - LCOV_BUILDER, - CHECKER_BUILDER, - AUDITOR_BUILDER_TEST, - AUDITOR_BUILDER_DEMO, - TEST_BUILDER, - TIP_RESERVE_TOPPER_BUILDER_DEMO, - TIP_RESERVE_TOPPER_BUILDER_TEST, - DEMO_SERVICES_CHECKER_BUILDER, - DOC_BUILDER, - SITES_BUILDER, - WALLET_BUILDER, - TALER_DEMO_HEALTHCHECK_BUILDER, - TALER_TEST_HEALTHCHECK_BUILDER, -] - -####### BUILDBOT SERVICES - -# 'services' is a list of BuildbotService items like reporter -# targets. The status of each build will be pushed to these -# targets. buildbot/reporters/*.py has a variety to choose from, -# like IRC bots. - -IRC = reporters.IRC( - "irc.eu.freenode.net", - "taler-bb", - useColors=False, - channels=[{ - "channel": "#taler" - }], - password="taler-bb-pass19", - notify_events={ - 'exception': 1, - 'successToFailure': 1, - 'failureToSuccess': 1 - } -) - -EMAIL = reporters.MailNotifier( - fromaddr="testbuild@taler.net", - sendToInterestedUsers=False, - # notify from pass to fail, and viceversa. - mode=("change"), - builders=( - "lcov-builder", - "checker-builder", - "doc-builder", - "test-builder", - "wallet-builder", - ## - # Rarely/never used, excluding. - # "selenium-builder" - ), - extraRecipients=["buildfailures@taler.net"], - subject="Taler build." -) - -c["services"] = [IRC, EMAIL] - -####### PROJECT IDENTITY - -c["title"] = "GNU Taler" -c["titleURL"] = "https://taler.net" - -# We use nginx to expose the BB under this URL. -c["buildbotURL"] = "https://buildbot.taler.net/" - - -BUILDER_LIST = [ - "linkchecker-builder", - "buildslavetest-builder", - "buildmaster-builder", - "doc-builder", - "lcov-builder", - "checker-builder", - "auditor-builder", - "auditor-builder-test", - "auditor-builder-demo", - "demo-services-checker-builder", - "tip-reserve-topper-builder-demo", - "tip-reserve-topper-builder-test", - "sites-builder", - "test-builder", - "wallet-builder", - "taler-demo-healthcheck-builder", - "taler-test-healthcheck-builder", -] - -# Authentication for Web UI. -exec(compile(open('../taler_bb_userpass_db/taler_bb_userpass_db/db.py').read(), filename='db.py', mode='exec')) - -authz = util.Authz( - allowRules=[ - util.ForceBuildEndpointMatcher(role="admins", - builder=b) - for b in BUILDER_LIST - ] + [ - util.StopBuildEndpointMatcher(role="admins", - builder=b) - for b in BUILDER_LIST - ] + [ - util.RebuildBuildEndpointMatcher(role="admins", - builder=b) - for b in BUILDER_LIST - ] + [ - util.ForceBuildEndpointMatcher(role="norole", - builder=b) - for b in BUILDER_LIST - ] + [ - util.StopBuildEndpointMatcher(role="norole", - builder=b) - for b in BUILDER_LIST - ] + [ - util.RebuildBuildEndpointMatcher(role="norole", - builder=b) - for b in BUILDER_LIST - ], - roleMatchers=[ - util.RolesFromUsername( - roles=["admins"], - usernames=["marcello", "florian", "christian", "buckE"] - ) - ] -) - -# minimalistic config to activate new web UI -# -- formerly commented out as not packaged properly in Debian and others, see -# https://bugzilla.redhat.com/show_bug.cgi?id=1557687 -# -- un-commented as "pip install buildbot.www" performed from buildbot-master account -c["www"] = { - "port": 8010, - "plugins": { - "waterfall_view": {}, - "console_view": {} - }, - "allowed_origins": ["https://*.taler.net"], - "avatar_methods": [], - "auth": util.UserPasswordAuth(USER_PASSWORD_DB), - "authz": authz -} - -####### DB URL - -c["db"] = { - # This specifies what database buildbot uses to store its - # state. You can leave this at its default for all but the - # largest installations. - "db_url": "sqlite:///state.sqlite", -} |