import re from getpass import getuser from buildbot.steps.source.git import Git from buildbot.steps.shell import ShellCommand from buildbot.plugins import (reporters, worker, changes, schedulers, util) # This is a sample buildmaster config file. It must be installed as # 'master.cfg' in your buildmaster's base directory. # This is the dictionary that the buildmaster pays attention to. # We also use a shorter alias to save typing. c = BuildmasterConfig = {} # Silence warning and allow very basic phoning home. c["buildbotNetUsageData"] = "basic" ####### WORKERS # The 'workers' list defines the set of recognized workers. Each # element is a Worker object, specifying a unique worker name and # password. The same worker name and password must be configured # on the worker. c["workers"] = [ ## # This worker makes the code coverage and publishes it # under the "lcov" Website. worker.Worker("lcov-worker", "lcov-pass"), ## # Undone. worker.Worker("selenium-worker", "selenium-pass"), ## # This worker builds manuals / API docs / tutorials. # worker.Worker("doc-worker", "doc-pass"), ## # This worker builds Websites: www/stage & docs-landing. # worker.Worker("sites-worker", "sites-pass"), ## # This worker builds Taler for the 'green' color under # the 'test' deployment. worker.Worker("builder-worker-green", "green-pass"), ## # This worker builds Taler for the 'blue' color under # the 'test' deployment. worker.Worker("builder-worker-blue", "blue-pass"), ## # This worker builds Taler for the 'green' color under # the 'test' deployment. worker.Worker("topper-worker-green", "topper-green-pass"), ## # This worker builds Taler for the 'blue' color under # the 'test' deployment. worker.Worker("topper-worker-blue", "topper-blue-pass"), ## # This worker cares about switching the 'socket' symlink # under /home/test; this way we activate the test or blue # color. worker.Worker("testswitcher-worker", "testswitcher-pass"), ## # Just a debug worker that responds to pushes made at help.git worker.Worker("debug-worker", "debug-pass"), ## # Undone worker.Worker("lint-worker", "lint-pass"), ## # This worker checks that all the services run under the # 'demo' deployment are up&running. worker.Worker("demo-worker", "demo-pass"), ## # Undone. worker.Worker("wallet-worker", "wallet-pass"), ## # This worker compiles the auditor reports for the "green" # deployment. worker.Worker("auditor-worker-green", "auditor-green-pass"), ## # This worker compiles the auditor reports for the "blue" # deployment. worker.Worker("auditor-worker-blue", "auditor-blue-pass"), ## # This worker compiles the auditor reports for the "green" # demo deployment. worker.Worker("demo-auditor-worker-green", "demo-auditor-green-pass"), ## # This worker compiles the auditor reports for the "blue" # demo deployment. worker.Worker("demo-auditor-worker-blue", "demo-auditor-blue-pass")] ## # Maps usernames to workers. # def pick_map(buildername): builder_worker_map = {"test-green": "builder-worker-green", "test-blue": "builder-worker-blue"} topper_worker_map = {"test-green": "topper-worker-green", "test-blue": "topper-worker-blue"} auditor_worker_map = { "test-green": "auditor-worker-green", "test-blue": "auditor-worker-blue", "demo-green": "demo-auditor-worker-green", "demo-blue": "demo-auditor-worker-blue"} meta_map = {"builder-builder": builder_worker_map, "auditor-builder": auditor_worker_map, "auditor-builder-demo": auditor_worker_map, "tip-reserve-topper-builder": topper_worker_map} return meta_map.get(buildername) # 'protocols' contains information about protocols which master # will use for communicating with workers. You must define at # least 'port' option that workers could connect to your master # with this protocol. 'port' must match the value configured into # the workers (with their --master option) c["protocols"] = { "pb": { "port": "tcp:9989:interface=127.0.0.1"}} ####### CHANGESOURCES # the 'change_source' setting tells the buildmaster how it should # find out about source code changes. # NOTE: BB is bound to localhost ALLCS = changes.PBChangeSource(user="allcs", passwd="allcs") c["change_source"] = [ALLCS] ####### SCHEDULERS # Configure the Schedulers, which decide how to react to incoming # changes. # Re-build documentation periodically def doc_filter(change): _change = change.asDict() if _change.get("project") in ["api", "merchant-frontend-examples"]: return True ## # All the other cases must involve a "doc" file in the # changes. files = _change.get("files") for file in files: if re.search(r"doc[s]?|manual", file.get("name", "")): return True ## # Must have been triggered by some change to C or Python. return False NIGHTLY_SCHEDULER = schedulers.Nightly( name="nightly-scheduler", builderNames=["lcov-builder", "auditor-builder-demo", "auditor-builder"], branch="master", hour=6, minute=0) DOC_SCHEDULER = schedulers.SingleBranchScheduler( name="periodic-doc-scheduler", builderNames=["doc-builder"], change_filter=util.ChangeFilter( branch_re="(master|stable)", filter_fn=doc_filter, project_re="(backoffice|api|www|bank|exchange|merchant" "|deployment|merchant-frontend-examples" "|docs-landing)"), treeStableTimer=None) SITES_SCHEDULER = schedulers.SingleBranchScheduler( name="sites-scheduler", builderNames=["sites-builder"], change_filter=util.ChangeFilter( branch_re="(master|stable)", project_re="(www|docs-landing)"), treeStableTimer=None) WALLET_SCHEDULER = schedulers.SingleBranchScheduler( name="wallet-scheduler", change_filter=util.ChangeFilter( branch="master", project_re="(wallet|deployment)"), treeStableTimer=None, builderNames=[]) DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic( name="demo-services-checker-scheduler", periodicBuildTimer=60*60, # 1 hour builderNames=["demo-services-checker-builder"]) TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic( name="tip-reserve-topper-scheduler", periodicBuildTimer=60*60*24*10, # 10 days builderNames=["tip-reserve-topper-builder"]) ALL_SCHEDULER = schedulers.SingleBranchScheduler( name="all-scheduler", change_filter=util.ChangeFilter( branch_re="(master|stable)", project_re="(backoffice|wallet|bank|exchange|" "merchant|deployment|donations|twister|" "blog|help|survey|landing|playground)"), treeStableTimer=None, builderNames=["builder-builder"]) TESTSWITCHER_SCHEDULER = schedulers.Dependent( name="testswitcher-scheduler", upstream=ALL_SCHEDULER, builderNames=["testswitcher-builder"]) # Scheduler monitoring the help.git repo; a forgotten repo we # use to test BB. DEBUG_SCHEDULER = schedulers.SingleBranchScheduler( name="debug-scheduler", change_filter=util.ChangeFilter( branch="master", project="help"), treeStableTimer=None, builderNames=["debug-builder"]) # Consider adding other Python parts, like the various frontends. LINT_SCHEDULER = schedulers.SingleBranchScheduler( name="lint-scheduler", change_filter=util.ChangeFilter( branch="master", project_re="(bank|donations|survey|blog)"), treeStableTimer=None, builderNames=["lint-builder"]) # Provide "force" button in the web UI. To be removed in the # future ? FORCE_SCHEDULER = schedulers.ForceScheduler( name="force-scheduler", builderNames=[ "debug-builder", "lcov-builder", "auditor-builder", "auditor-builder-demo", "selenium-builder", "testswitcher-builder", "builder-builder", "doc-builder", "sites-builder", "wallet-builder", "tip-reserve-topper-builder"]) c["schedulers"] = [ NIGHTLY_SCHEDULER, TIP_RESERVE_TOPPER_SCHEDULER, DEMO_SERVICES_CHECKER_SCHEDULER, DOC_SCHEDULER, SITES_SCHEDULER, WALLET_SCHEDULER, ALL_SCHEDULER, TESTSWITCHER_SCHEDULER, FORCE_SCHEDULER, DEBUG_SCHEDULER, LINT_SCHEDULER] ####### BUILDERS # The 'builders' list defines the Builders, which tell Buildbot # how to perform a build: what steps, and which workers can execute # them. Note that any particular build will only take place on # one worker. def git_step(repo): return Git(repourl=repo, mode="full", method="fresh", logEnviron=False, alwaysUseLatest=True, haltOnFailure=True, branch="master") WALLET_FACTORY = util.BuildFactory() WALLET_FACTORY.addStep(git_step( "git://git.taler.net/wallet-webex.git")) WALLET_FACTORY.addStep(ShellCommand( name="configuration", description="Running configure script", descriptionDone="Correctly configured", command=["./configure"], workdir="build/")) WALLET_FACTORY.addStep(ShellCommand( name="test", description="Running wallet tests", descriptionDone="Test correctly run", command=["make", "check"], workdir="build/")) WALLET_FACTORY.addStep(ShellCommand( name="lint", description="Linting the wallet", descriptionDone="Linting done", command=["make", "lint"], workdir="build/")) DEBUG_FACTORY = util.BuildFactory() DEBUG_FACTORY.addStep(ShellCommand( name="echo debug", description="just echoing a word", descriptionDone="builder responded", command=["echo", "I'm here!"])) def lint_dispatcher(project): return "./lint_%s.sh" % project LINT_FACTORY = util.BuildFactory() LINT_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) LINT_FACTORY.addStep(ShellCommand( name="Python linter", description="linting Python", descriptionDone="linting done", command=util.Transform(lint_dispatcher, util.Property("project")), workdir="build/taler-build")) LCOV_FACTORY = util.BuildFactory() LCOV_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) LCOV_FACTORY.addStep(ShellCommand( haltOnFailure=True, name="invalidation", description="Invalidating timestamps", descriptionDone="timestamps invalidated", command=["./invalidate.sh"], workdir="build/taler-build", env={"TALER_ENV_NAME": "not-test"})) # work-around 'set -eu' LCOV_FACTORY.addStep(ShellCommand( name="build", description="Compiling..", descriptionDone="lcov files generated", command=["make", "lcov"], workdir="build/taler-build", env={"PATH": "${HOME}/local/bin:${PATH}", "TALER_CHECKDB": "postgres:///talercheck-${USER}"})) # FIXME: 'demo' reports generator missing. AUDITOR_FACTORY = util.BuildFactory() AUDITOR_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) AUDITOR_FACTORY.addStep(ShellCommand( name="Auditor reports generator", description="Generating auditor reports.", descriptionDone="Auditor reports correctly generated.", command=["./make_auditor_reports.sh"], workdir="build/buildbot")) TIP_RESERVE_TOPPER_FACTORY = util.BuildFactory() TIP_RESERVE_TOPPER_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) TIP_RESERVE_TOPPER_FACTORY.addStep(ShellCommand( name="tip reserve topper", description="Topping the tip reserve.", descriptionDone="Tip reserve has been topped.", command=["./top_reserve.sh"], workdir="build/buildbot")) BUILD_FACTORY = util.BuildFactory() BUILD_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) BUILD_FACTORY.addStep(ShellCommand( name="build", description="Building inactive blue-green party.", descriptionDone="Inactive party got built.", command=["./build.sh"], workdir="build/buildbot", haltOnFailure=True)) BUILD_FACTORY.addStep(ShellCommand( name="config", description="Generating configuration file.", descriptionDone="Configuration file generated.", command=["./config.sh"], workdir="build/buildbot", haltOnFailure=True)) BUILD_FACTORY.addStep(ShellCommand( name="keys generation and sign", description="Generating exchange keys, and auditor-sign them.", descriptionDone="Exchange keys generated, and auditor-signed.", command=["./keys.sh"], workdir="build/buildbot", haltOnFailure=True, env={'BRANCH': util.Property("branch")})) BUILD_FACTORY.addStep(ShellCommand( name="wire details sign", description="Signing exchange wire details.", descriptionDone="Exchange wire details got signed.", command=["./sign.sh"], workdir="build/buildbot", haltOnFailure=True, env={'BRANCH': util.Property("branch")})) BUILD_FACTORY.addStep(ShellCommand( name="restart services", description="Restarting inactive blue-green party.", descriptionDone="Restarting Taler.", command=["./restart.sh"], workdir="build/buildbot", haltOnFailure=True, env={'BRANCH': util.Property("branch")})) BUILD_FACTORY.addStep(ShellCommand( name="check services correctly restarted", description="Checking services are correctly restarted.", descriptionDone="All services are correctly restarted.", command=["./checks.sh"], workdir="build/buildbot", haltOnFailure=True, env={'DEPLOYMENT': "test"})) TESTSWITCH_FACTORY = util.BuildFactory() TESTSWITCH_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) TESTSWITCH_FACTORY.addStep(ShellCommand( name="switch active party", description="Switch to the party which was inactive.", descriptionDone="Active party has been switched.", command=["./switch.sh"], workdir="build/buildbot")) SELENIUM_FACTORY = util.BuildFactory() SELENIUM_FACTORY.addStep(ShellCommand( name="selenium", description="Headless browser test", descriptionDone="Test finished", command=["launch_selenium_test"], env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"})) DOC_FACTORY = util.BuildFactory() DOC_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) DOC_FACTORY.addStep(ShellCommand( name="build docs", description="Building documentation", descriptionDone="Documentation built.", command=["./build-docs.sh"], workdir="build/buildbot", haltOnFailure=True)) SITES_FACTORY = util.BuildFactory() SITES_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) SITES_FACTORY.addStep(ShellCommand( name="build Web sites", description="Building all the Taler homepages", descriptionDone="Sites built.", command=["./build-sites.sh"], workdir="build/buildbot", haltOnFailure=True)) DEMO_SERVICES_CHECKER_FACTORY = util.BuildFactory() DEMO_SERVICES_CHECKER_FACTORY.addStep(git_step( "git://git.taler.net/deployment.git")) DEMO_SERVICES_CHECKER_FACTORY.addStep(ShellCommand( name="demo services checker", description="Checking demo services are online", descriptionDone="Demo services are online!.", command=["./checks.sh"], workdir="build/buildbot", haltOnFailure=True, # Needed to test the 'demo' deployment. env={"DEPLOYMENT": "demo"})) DEBUG_BUILDER = util.BuilderConfig( name="debug-builder", workernames=["debug-worker"], factory=DEBUG_FACTORY) LINT_BUILDER = util.BuilderConfig( name="lint-builder", workernames=["lint-worker"], factory=LINT_FACTORY) LCOV_BUILDER = util.BuilderConfig( name="lcov-builder", workernames=["lcov-worker"], factory=LCOV_FACTORY) ## # Helper function that picks the active deployment between # blue and green and schedules it as the next worker for producing # auditor reports. def pick_active_color( marker, builder, workers_list, build_request): f = open(marker, "r") if not f: print("No active color in marker (%s)" % marker) f.close() return None active = f.readline().rstrip() print("Found active color: %s" % str(active)) aw = pick_map(builder.name).get(str(active)) for worker in workers_list: if aw == worker.worker.workername: f.close() print("Scheduling worker: %s" % str(worker)) return worker print("Couldn't find the active worker, aborting.") f.close() return None def pick_active_color_test (builder, workers_list, build_request): return pick_active_color ( "/home/test/active", builder, workers_list, build_request) def pick_active_color_demo (builder, workers_list, build_request): return pick_active_color ( "/home/demo/active", builder, workers_list, build_request) AUDITOR_BUILDER = util.BuilderConfig( name="auditor-builder", nextWorker=pick_active_color_test, workernames=["auditor-worker-blue", "auditor-worker-green"], factory=AUDITOR_FACTORY) AUDITOR_BUILDER_DEMO = util.BuilderConfig( name="auditor-builder-demo", nextWorker=pick_active_color_demo, workernames=["demo-auditor-worker-blue", "demo-auditor-worker-green"], factory=AUDITOR_FACTORY) TIP_RESERVE_TOPPER_BUILDER = util.BuilderConfig( name="tip-reserve-topper-builder", workernames=["topper-worker-green", "topper-worker-blue"], factory=TIP_RESERVE_TOPPER_FACTORY, nextWorker=pick_active_color) DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig( name="demo-services-checker-builder", workernames="demo-worker", factory=DEMO_SERVICES_CHECKER_FACTORY) ## # Helper function that picks the inactive deployment between # blue and green and schedules it as the next worker for building # Taler. def pick_inactive_color(builder, workers_list, build_request): f = open("/home/test/nonactive", "r") if not f: print("'test' doesn't specify the nonactive color, abort") f.close() return None nonactive = f.readline().rstrip() print("Found nonactive color: %s" % str(nonactive)) nw = pick_map(builder.name).get(str(nonactive)) for worker in workers_list: if nw == worker.worker.workername: f.close() print("Scheduling worker: %s" % str(worker)) return worker print("Couldn't find the inactive worker, aborting.") f.close() return None BUILDER_BUILDER = util.BuilderConfig( name="builder-builder", workernames=["builder-worker-green", "builder-worker-blue"], nextWorker=pick_inactive_color, factory=BUILD_FACTORY) TESTSWITCHER_BUILDER = util.BuilderConfig( name="testswitcher-builder", workernames=["testswitcher-worker"], factory=TESTSWITCH_FACTORY) SELENIUM_BUILDER = util.BuilderConfig( name="selenium-builder", workernames=["selenium-worker"], factory=SELENIUM_FACTORY) DOC_BUILDER = util.BuilderConfig( name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY) SITES_BUILDER = util.BuilderConfig( name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY) WALLET_BUILDER = util.BuilderConfig( name="wallet-builder", workernames=["wallet-worker"], factory=WALLET_FACTORY) c["builders"] = [ LCOV_BUILDER, AUDITOR_BUILDER, AUDITOR_BUILDER_DEMO, BUILDER_BUILDER, TESTSWITCHER_BUILDER, TIP_RESERVE_TOPPER_BUILDER, DEMO_SERVICES_CHECKER_BUILDER, SELENIUM_BUILDER, DOC_BUILDER, SITES_BUILDER, WALLET_BUILDER, DEBUG_BUILDER, LINT_BUILDER] ####### BUILDBOT SERVICES # 'services' is a list of BuildbotService items like reporter # targets. The status of each build will be pushed to these # targets. buildbot/reporters/*.py has a variety to choose from, # like IRC bots. IRC = reporters.IRC( "irc.eu.freenode.net", "taler-bb", useColors=False, channels=[{"channel": "#taler"}], password="taler-bb-pass19", notify_events={ 'exception': 1, 'successToFailure': 1, 'failureToSuccess': 1}) EMAIL = reporters.MailNotifier( fromaddr="testbuild@taler.net", sendToInterestedUsers=False, # notify from pass to fail, and viceversa. mode=("change"), builders=( "lcov-builder", "testswitcher-builder", "doc-builder", "wallet-builder", "selenium-builder"), extraRecipients=["buildfailures@taler.net"], subject="Taler build.") c["services"] = [IRC, EMAIL] ####### PROJECT IDENTITY c["title"] = "Taler" c["titleURL"] = "https://taler.net" # We use nginx to expose the BB under this URL. c["buildbotURL"] = "https://buildbot.taler.net/" from taler_bb_userpass_db import USER_PASSWORD_DB BUILDER_LIST = ["testswitcher-builder", "doc-builder", "lcov-builder", "wallet-builder", "auditor-builder", "auditor-builder-demo", "demo-services-checker-builder", "tip-reserve-topper-builder", "lint-builder", "selenium-builder"] authz = util.Authz( allowRules=[ util.ForceBuildEndpointMatcher( role="admins", builder=b) for b in BUILDER_LIST] + [util.StopBuildEndpointMatcher( role="admins", builder=b) for b in BUILDER_LIST] + [util.RebuildBuildEndpointMatcher( role="admins", builder=b) for b in BUILDER_LIST] + [util.ForceBuildEndpointMatcher( role="norole", builder=b) for b in BUILDER_LIST] + [util.StopBuildEndpointMatcher( role="norole", builder=b) for b in BUILDER_LIST] + [util.RebuildBuildEndpointMatcher( role="norole", builder=b) for b in BUILDER_LIST], roleMatchers=[ util.RolesFromUsername(roles=["admins"], usernames=["marcello", "florian", "christian"])]) # minimalistic config to activate new web UI c["www"] = { "port": 8010, "plugins" : { "waterfall_view": {}, "console_view":{}}, "allowed_origins": ["https://*.taler.net"], "avatar_methods": [], "auth": util.UserPasswordAuth(USER_PASSWORD_DB), "authz": authz} ####### DB URL c["db"] = { # This specifies what database buildbot uses to store its # state. You can leave this at its default for all but the # largest installations. "db_url" : "sqlite:///state.sqlite", }