diff options
Diffstat (limited to 'buildbot/master.cfg')
| -rw-r--r-- | buildbot/master.cfg | 1213 |
1 files changed, 368 insertions, 845 deletions
diff --git a/buildbot/master.cfg b/buildbot/master.cfg index 24f5483..e6f2e78 100644 --- a/buildbot/master.cfg +++ b/buildbot/master.cfg @@ -1,6 +1,9 @@ +# -*- python -*- +# ex: set syntax=python: + ## # This file is part of TALER -# (C) 2016-2022 Taler Systems SA +# (C) 2016-2023 Taler Systems SA # # TALER is free software; you can redistribute it and/or # modify it under the terms of the GNU Affero General Public @@ -20,12 +23,28 @@ # @author Marcello Stanisci # @author ng0 # @author Christian Grothoff +# @author Devan Carpenter +import ast +import configparser +import glob +import os +import pathlib +import pwd import re -from getpass import getuser +import subprocess + +from buildbot.changes.pb import PBChangeSource from buildbot.steps.source.git import Git from buildbot.steps.shell import ShellCommand -from buildbot.plugins import * +from buildbot.plugins import changes +from buildbot.plugins import reporters +from buildbot.plugins import schedulers +from buildbot.plugins import steps +from buildbot.plugins import util +from buildbot.process import buildstep, logobserver from buildbot.reporters.generators.build import BuildStatusGenerator +from buildbot.worker import Worker +from twisted.internet import defer # This is a sample buildmaster config file. It must be # installed as 'master.cfg' in your buildmaster's base @@ -68,19 +87,20 @@ class MessageFormatterWithStdout(reporters.MessageFormatter): stdout.append(line[1:]) ctx.update(dict(stdout="\n".join(stdout))) -tipReserveEmails = reporters.MailNotifier( - fromaddr="buildbot@taler.net", # to be sent to a dedicate alias - sendToInterestedUsers=False, - mode=("all"), - builders=["check-tips-builder"], - extraRecipients=["tips@taler.net"], - dumpMailsToLog=True, # debug, to remove - messageFormatter=MessageFormatterWithStdout( - wantSteps=True, - wantLogs=True, - template="{{ stdout }}", - subject="tips availability on demo") -) +# tipReserveEmails = reporters.MailNotifier( +# fromaddr="buildbot@taler.net", # to be sent to a dedicate alias +# sendToInterestedUsers=False, +# mode=("all"), +# builders=["check-tips-builder"], # This builder has been removed - Javisep. +# extraRecipients=["tips@taler.net"], +# dumpMailsToLog=True, # debug, to remove +# messageFormatter=MessageFormatterWithStdout( +# wantSteps=True, +# wantLogs=True, +# template="{{ stdout }}", +# subject="tips availability on demo") +# ) + SERVICES = [] @@ -98,27 +118,32 @@ SCHEDULERS = [] NIGHTLY_TRIGGERS=[] # Array of builders to be scheduled whenever any of the code Git repos change -CODECHANGE_TRIGGERS=[] +CODECHANGE_TRIGGERS = [] -# Array of builders to be scheduled whenever the wallet-core or deployment change -WALLETCHANGE_TRIGGERS=[] +# Array of builders to be scheduled whenever the wallet-core or +# deployment change +WALLETCHANGE_TRIGGERS = [] -# Array of builder names for which build status reports should be sent via e-mail -EMAIL_ALERTS=[] +# Array of builder names for which build status reports should be sent +# via e-mail +EMAIL_ALERTS = [] +# Array of email address for which build status reports shoudl be sent +BUILDER_EMAIL_ADDRESSES = [] ############ Convenience functions ################# # Create a FACTORY with a deployment.git checkout as the first step. def create_factory_with_deployment(): f = util.BuildFactory() - update_deployment (f) + update_deployment(f) return f + # Convenience function that checks out a Git repository. # First argument is the URL of the Git to clone, second # the desired branch. Default is 'master'. -def git_step(repo,target_branch="master"): +def git_step(repo, target_branch="master"): return Git( repourl=repo, mode="full", @@ -129,9 +154,10 @@ def git_step(repo,target_branch="master"): branch=target_branch ) + # Convenience function that runs 'make check' in a # directory of the code inside of a netjail. -def jailed_check(package,srcdirs): +def jailed_check(package, srcdirs): return steps.ShellSequence( name="Tests of " + package, description="Testing " + package, @@ -141,6 +167,7 @@ def jailed_check(package,srcdirs): workdir="../../sources/" + package ) + # Convenience function that checks out the deployment. def update_deployment(factory): factory.addStep(steps.ShellSequence( @@ -156,6 +183,69 @@ def update_deployment(factory): )) +# Convenience function that builds and runs a container. +def container_add_step(HALT_ON_FAILURE, + WARN_ON_FAILURE, + CONTAINER_BUILD, + CONTAINER_NAME, + factory, + WORK_DIR, + stepName, + CONTAINER_ARCH="amd64", + jobCmd="/workdir/contrib/ci/ci.sh", + containerFile="contrib/ci/Containerfile"): + print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") + if not CONTAINER_BUILD: + return steps.ShellSequence( + name=stepName, + commands=[ + util.ShellArg(command=["podman", "run", "--rm", + "--arch", CONTAINER_ARCH, + "--add-host", "taler.host.internal:10.0.2.2", + "--network", "slirp4netns:allow_host_loopback=true", + "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), + "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), + "--volume", f"{WORK_DIR}:/workdir", + "--volume", "/home/container-worker/container_artifacts:/artifacts", + "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", + "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", + "--workdir", "/workdir", + CONTAINER_NAME, jobCmd], + logname='run inside container', + haltOnFailure=HALT_ON_FAILURE), + ], + haltOnFailure=HALT_ON_FAILURE, + workdir=WORK_DIR + ) + else: + return steps.ShellSequence( + name=stepName, + commands=[ + util.ShellArg(command=["podman", "build", "-t", CONTAINER_NAME, + "--arch", CONTAINER_ARCH, + "-f", containerFile, "."], + logname='build container', haltOnFailure=True), + util.ShellArg(command=["podman", "run", "--rm", + "--arch", CONTAINER_ARCH, + "--add-host", "taler.host.internal:10.0.2.2", + "--network", "slirp4netns:allow_host_loopback=true", + "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), + "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), + "--volume", f"{WORK_DIR}:/workdir", + "--volume", "/home/container-worker/container_artifacts:/artifacts", + "--volume", f"/run/user/{pwd.getpwnam('container-worker').pw_uid}/podman/podman.sock:/run/podman/podman.sock", + "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", + "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", + "--security-opt", "label=disable", + "--workdir", "/workdir", + CONTAINER_NAME, jobCmd], + logname='run inside container', + haltOnFailure=HALT_ON_FAILURE), + ], + haltOnFailure=HALT_ON_FAILURE, + workdir=WORK_DIR + ) + ################################################################## ######################## JOBS #################################### ################################################################## @@ -171,11 +261,12 @@ def update_deployment(factory): ################ 1: BUILDMASTER JOB ################################### + ## # This worker restarts the buildmaster itself on # changes to this file. # Location: /home/buildbot-master @ taler.net -WORKERS.append(worker.Worker("buildmaster-worker", "buildmaster-pass")) +WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) BUILDMASTER_FACTORY = create_factory_with_deployment() BUILDMASTER_FACTORY.addStep( @@ -208,115 +299,6 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( )) -################ 2: DOCUMENTATION JOB ################################### - -## -# This worker builds manuals / API docs / tutorials. -# Location: /home/docbuilder @ taler.net -WORKERS.append(worker.Worker("doc-worker", "doc-pass")) - -DOC_FACTORY = create_factory_with_deployment() -DOC_FACTORY.addStep( - ShellCommand( - name="build docs", - description="Building documentation", - descriptionDone="Documentation built.", - command=["./build-docs.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) -DOC_FACTORY.addStep( - steps.ShellSequence( - name="prepare exchange", - description="Running bootstrap and configure for exchange", - descriptionDone="exchange ready for doxygen", - commands=[ - util.ShellArg(command=["./bootstrap"], logname='bootstrap'), - util.ShellArg(command=["./configure", "--enable-only-doc"], logname='configure'), - ], - workdir="../../sources/exchange/", - haltOnFailure=True, - ) -) -DOC_FACTORY.addStep( - ShellCommand( - name="doxygen::exchange", - description="building exchange doxygen documentation", - descriptionDone="doxygen on exchange finished", - command=["make", "full" ], - want_stderr=False, - workdir="../../sources/exchange/doc/doxygen", - haltOnFailure=True, - ) -) -DOC_FACTORY.addStep( - steps.ShellSequence( - name="prepare merchant", - description="prepare merchant", - descriptionDone="merchant prepared", - commands=[ - util.ShellArg(command=["./bootstrap"], logname='bootstrap'), - util.ShellArg(command=["./configure", "--enable-only-doc"], logname='configure'), - util.ShellArg(command=["cp", "../exchange/doc/doxygen/taler-exchange.tag", "doc/doxygen/taler-exchange.tag"]), - ], - workdir="../../sources/merchant/" - ) -) -DOC_FACTORY.addStep( - ShellCommand( - name="doxygen::merchant", - description="building merchant doxygen documentation", - descriptionDone="doxygen on merchant finished", - command=["make", "full" ], - want_stderr=False, - workdir="../../sources/merchant/doc/doxygen" - ) -) -DOC_FACTORY.addStep( - steps.ShellSequence( - name="doxygen::wallet", - description="building wallet typescript documentation", - descriptionDone="typedoc on taler-wallet-core finished", - commands=[ - util.ShellArg(command=["./bootstrap"], logname="bootstrap"), - util.ShellArg(command=["./configure"], logname="configure"), - util.ShellArg(command=["make"], logname="make"), - util.ShellArg(command=["pnpm", "install", "-W", "typedoc"], logname="pnpm"), - util.ShellArg(command=["./node_modules/typedoc/bin/typedoc", "--out", "dist/typedoc", "--tsconfig", "tsconfig.build.json", "packages/taler-util/src/index.ts", "packages/taler-wallet-cli/src/index.ts", "packages/taler-wallet-android/src/index.ts", "packages/taler-wallet-core/src/index.ts" ], logname="typedoc"), - ], - workdir="../../sources/wallet-core/" - ) -) - - -BUILDERS.append(util.BuilderConfig( - name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY -)) - -EMAIL_ALERTS.append("doc-builder") - -#sphinxErrorNotifier = reporters.MailNotifier( -# fromaddr="bb@taler.net", -# sendToInterestedUsers=False, -# addLogs=['build docs.stdio',], -# useTls=True, -# # notify if sphinx exits with error (command line option in Makefile turns warnings into exit 1) -# mode=('failing'), -# builders=('doc-builder',), -# extraRecipients=['sphinxerrors@taler.net'] -#) - - -# Docs run if master or stable branch of 'docs' (or deployment) changed. -SCHEDULERS.append(schedulers.SingleBranchScheduler( - name="periodic-doc-scheduler", - builderNames=["doc-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", project_re="(docs|deployment)" - ), - treeStableTimer=None, -)) ################ 3: WEBSITE JOB ################################### @@ -324,7 +306,7 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( ## # This worker builds Websites: www and stage. # -WORKERS.append(worker.Worker("sites-worker", "sites-pass")) +WORKERS.append(Worker("sites-worker", "sites-pass")) SITES_FACTORY = create_factory_with_deployment() SITES_FACTORY.addStep( @@ -342,7 +324,8 @@ BUILDERS.append(util.BuilderConfig( name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY )) -EMAIL_ALERTS.append("sites-builder") +#EMAIL_ALERTS.append("sites-builder") + # The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed: def twister_web_page(change): @@ -358,6 +341,7 @@ def twister_web_page(change): return True return False + # Sites are re-build whenever deployment, www buywith, or twister changes. SCHEDULERS.append(schedulers.SingleBranchScheduler( name="sites-scheduler", @@ -370,345 +354,12 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( )) -################ 4: LCOV JOB ################################### - -## -# This worker makes the code coverage and publishes it -# under the "lcov" Website. -WORKERS.append(worker.Worker("lcov-worker", "lcov-pass")) - -LCOV_FACTORY = create_factory_with_deployment() -LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -LCOV_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install of wallet", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"], - workdir="build/", - haltOnFailure=True, - ) -) -LCOV_FACTORY.addStep( - ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'TALER_COVERAGE': "1"}, - ) -) -LCOV_FACTORY.addStep( - ShellCommand( - name="coverage generation", - description="running tests", - descriptionDone="generating HTML report", - command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "/home/lcovworker/deployment/buildbot/coverage.sh"], - workdir="../../deployment/buildbot", - env={'PATH': "${HOME}/local/bin:${PATH}"} - ) -) - -BUILDERS.append(util.BuilderConfig( - name="lcov-builder", - workernames=["lcov-worker"], - factory=LCOV_FACTORY -)) - -EMAIL_ALERTS.append("lcov-builder") -NIGHTLY_TRIGGERS.append("lcov-builder") -NIGHTLY_TRIGGERS.append("check-tips-builder") -NIGHTLY_TRIGGERS.append("taler-demo-healthcheck-builder") - -################ 5: UNIT TEST JOB ################################### - -## -# This worker builds everything and runs our 'make check' -# test suite against 'everything'. -WORKERS.append(worker.Worker("checker-worker", "checker-pass")) - -INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch gnunet.org sources", - description="fetching latest deployment repositories from git.gnunet.org", - descriptionDone="GNUnet code base updated", - command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch taler.net sources", - description="fetching latest deployment repositories from git.taler.net", - descriptionDone="Taler code base updated", - command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "bank", "twister"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - ) -) -INTEGRATIONTEST_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install of wallet", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"], - workdir="../../sources/wallet-core", - haltOnFailure=True, - ) -) -INTEGRATIONTEST_FACTORY.addStep( - steps.ShellSequence( - name="fetch", - description="Running local install of wallet", - descriptionDone="local wallet install done", - commands=[ - util.ShellArg(command=["./bootstrap"]), - util.ShellArg(command=["./configure", "--prefix=$HOME/local/"]), - util.ShellArg(command=["make install"]), - ], - workdir="../../sources/wallet-core", - haltOnFailure=True, - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="build", - description="Building other Taler components", - descriptionDone="Taler built", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=False - ) -) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", [ - "microhttpd", - "testcurl", - "testzzuf", -])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", [ - "util", - "pq", - "curl", - "json", -])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("twister", [ - "test", -])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", [ - "util", - "curl", - "mhd", - "pq", - "json", - "bank-lib", - "exchangedb", - "auditordb", - "exchange", - "auditor", - "lib", - "exchange-tools", - "testing", - "benchmark", -])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", [ - "mustach", - "backenddb", - "backend", - "lib", - "testing", - "merchant-tools", -])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", [ - "util", - "syncdb", - "sync", - "lib" -])) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="bank check", - description="Testing Taler Python bank", - descriptionDone="Done", - command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"], - workdir="../../sources/bank", - haltOnFailure=False, - env={'PYTHONUSERBASE': "$HOME/local" } - ) -) -INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="wallet check", - description="Testing wallet-core", - descriptionDone="Done", - command=["make", "check"], - workdir="../../sources/wallet-core", - haltOnFailure=False - ) -) - -BUILDERS.append(util.BuilderConfig( - name="checker-builder", - workernames=["checker-worker"], - factory=INTEGRATIONTEST_FACTORY -)) - -EMAIL_ALERTS.append("checker-builder") -CODECHANGE_TRIGGERS.append("checker-builder") - - -################ 6: 'test.taler.net' deployment JOB ################################### - -## -# This worker builds Taler for the 'test' deployment. -WORKERS.append(worker.Worker("test-worker", "test-pass")) - -# buildslavetest FACTORY -BUILDSLAVETEST_FACTORY = create_factory_with_deployment() -BUILDSLAVETEST_FACTORY.addStep( - ShellCommand( - name="buildslavetest script (for testing purposes)", - description="Build Slave Test", - descriptionDone="buildslavetest: Done", - command=["./buildslavetest.sh"], - workdir="/home/buildslavetest/" - ) -) - -# buildslavetest BUILDER -BUILDERS.append(util.BuilderConfig( - name="buildslavetest-builder", - workernames=["buildslavetest-worker"], - factory=BUILDSLAVETEST_FACTORY -)) - -EMAIL_ALERTS.append("buildslavetest-builder") - -# buildslavetest SCHEDULER -SCHEDULERS.append(schedulers.SingleBranchScheduler( - name="buildslavetest-scheduler", - builderNames=["buildslavetest-builder"], - change_filter=util.ChangeFilter( - branch_re="(master|stable)", project_re="(help|deployment)" - ), - treeStableTimer=None, -)) - -################ 7: 'test-auditor' deployment JOB ################################### - - -## -# This worker compiles the auditor reports for the "test" -# demo deployment. -WORKERS.append(worker.Worker("test-auditor-worker", "test-auditor-pass")) - -AUDITOR_FACTORY_TEST = create_factory_with_deployment() -AUDITOR_FACTORY_TEST.addStep( - ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/taler-test/"} - ) -) - -BUILDERS.append(util.BuilderConfig( - name="auditor-builder-test", - workernames=["test-auditor-worker"], - factory=AUDITOR_FACTORY_TEST -)) -NIGHTLY_TRIGGERS.append("auditor-builder-test") - - -################ 8: 'demo-auditor' deployment JOB ################################### -## -# This worker compiles the auditor reports for the "green" -# demo deployment. -WORKERS.append(worker.Worker("demo-auditor-worker", "demo-auditor-pass")) - -AUDITOR_FACTORY_DEMO = create_factory_with_deployment() -AUDITOR_FACTORY_DEMO.addStep( - ShellCommand( - name="Auditor reports generator", - description="Generating auditor reports.", - descriptionDone="Auditor reports correctly generated.", - command=["./make_auditor_reports.sh"], - workdir="../../deployment/buildbot", - env={'TALER_HOME': "/home/demo/active-home/"} - ) -) - -BUILDERS.append(util.BuilderConfig( - name="auditor-builder-demo", - workernames=["demo-auditor-worker"], - factory=AUDITOR_FACTORY_DEMO -)) - -NIGHTLY_TRIGGERS.append("auditor-builder-demo") - - -################ 8: 'build wallet-core' JOB ################################### - -## -# This worker builds wallet-core. -WORKERS.append(worker.Worker("wallet-worker", "wallet-pass")) - -WALLET_FACTORY = create_factory_with_deployment() -WALLET_FACTORY.addStep( - ShellCommand( - name="build", - description="Building all Taler codebase.", - descriptionDone="Taler built.", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) -WALLET_FACTORY.addStep( - ShellCommand( - name="test", - description="Running wallet tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", - "taler-wallet-cli", "testing", "run-integrationtests", - "--suites", "wallet,merchant,libeufin,wallet-backup,wallet-tipping"], - workdir="../../", - env={'PATH': "${HOME}/local/bin:${PATH}"} - ) -) - -BUILDERS.append(util.BuilderConfig( - name="wallet-builder", - workernames=["wallet-worker"], - factory=WALLET_FACTORY -)) - -EMAIL_ALERTS.append("wallet-builder") - -# Wallet is re-build whenever wallet-core or deployment changes -SCHEDULERS.append(schedulers.SingleBranchScheduler( - name="wallet-scheduler", - change_filter=util.ChangeFilter( - branch="master", - project_re="(wallet-core|deployment|libeufin)" - ), - treeStableTimer=None, - builderNames=["wallet-builder"] -)) - - ################ 9: 'check links' JOB ################################### ## # linkchecker worker checks for dead links in the Website # Location: /home/linkchecker @ taler.net -WORKERS.append(worker.Worker("linkchecker-worker", "linkchecker-pass")) +WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) # linkchecker FACTORY LINKCHECKER_FACTORY = create_factory_with_deployment() @@ -720,6 +371,7 @@ LINKCHECKER_FACTORY.addStep( command=["/home/linkchecker/deployment/buildbot/linkchecker.sh"], workdir="/home/linkchecker", haltOnFailure=True, + timeout=7200 # 2 hours ) ) @@ -732,7 +384,7 @@ BUILDERS.append(util.BuilderConfig( )) docs_generator = BuildStatusGenerator( - mode=('change','problem','failing','exception',), + mode=('change', 'problem', 'failing', 'exception',), builders=[ 'linkchecker-builder', ], @@ -741,388 +393,261 @@ docs_generator = BuildStatusGenerator( wantSteps=True, wantLogs=True ), - add_logs=True + add_logs=True ) + SERVICES.append(reporters.MailNotifier( fromaddr="bb@taler.net", generators=[docs_generator], sendToInterestedUsers=False, - useTls=True, + useTls=False, + relayhost="localhost", + smtpPort=25, + dumpMailsToLog=True, extraRecipients=['linkcheck@taler.net'] )) -SERVICES.append(tipReserveEmails) - -NIGHTLY_TRIGGERS.append("linkchecker-builder") +# SERVICES.append(tipReserveEmails) -################ 10: 'check spelling' JOB ################################### - -## -# codespell worker checks for spelling mistakes in code -# Location: /home/codespell @ taler.net -WORKERS.append(worker.Worker("codespell-worker", "codespell-pass")) - -CODESPELL_FACTORY = create_factory_with_deployment() -CODESPELL_FACTORY.addStep( - ShellCommand( - name="clean old deployment", - description="cleaning previous doxygen runs", - descriptionDone="Doxygen cleaned", - command=["rm", "-rf", "exchange/doc/doxygen/html/", "merchant/doc/doxygen/html/" ], - workdir="/home/codespell/sources/" - ) -) -CODESPELL_FACTORY.addStep( - ShellCommand( - name="fetch gnunet.org sources", - description="fetching latest deployment repositories from git.gnunet.org", - descriptionDone="GNUnet code base updated", - command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - ) -) -CODESPELL_FACTORY.addStep( - ShellCommand( - name="fetch taler.net sources", - description="fetching latest deployment repositories from git.taler.net", - descriptionDone="Taler code base updated", - command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "bank", "twister"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - ) -) -CODESPELL_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -CODESPELL_FACTORY.addStep( - ShellCommand( - name="codespell", - description="spell checking repositories", - descriptionDone="Spell check complete", - command=["/home/codespell/.local/bin/codespell", "-I", "/home/codespell/deployment/codespell/dictionary.txt", "-S", "*.bib,*.bst,*.cls,*.json,*.png,*.svg,*.wav,*.gz,*/templating/test?/**,**/auditor/*.sql,**/templating/mustach**,*.fees,*key,*.tag,*.info,*.latexmkrc,*.ecc,*.jpg,*.zkey,*.sqlite,*/contrib/hellos/**,*/vpn/tests/**,*.priv,*.file,*.tgz,*.woff,*.gif,*.odt,*.fee,*.deflate,*.dat,*.jpeg,*.eps,*.odg,*/m4/ax_lib_postgresql.m4,*/m4/libgcrypt.m4,*.rpath,config.status,ABOUT-NLS,*/doc/texinfo.tex,*.PNG,*.??.json,*.docx,*.ods,*.doc,*.docx,*.xcf,*.xlsx,*.ecc,*.ttf,*.woff2,*.eot,*.ttf,*.eot,*.mp4,*.pptx,*.epgz,*.min.js,**/*.map,**/fonts/**,*.pack.js,*.po,*.bbl,*/afl-tests/*,*/.git/**,*.pdf,*.epub,**/signing-key.asc,**/pnpm-lock.yaml,**/*.svg,**/*.cls,**/rfc.bib,**/*.bst,*/cbdc-es.tex,*/cbdc-it.tex,**/ExchangeSelection/example.ts,*/testcurl/test_tricky.c,*/i18n/strings.ts,*/src/anastasis-data.ts", "bank", "exchange", "libmicrohttpd", "merchant", "sync", "twister", "wallet-core"], - workdir="/home/codespell/sources/" - ) -) -CODESPELL_FACTORY.addStep( - ShellCommand( - name="gana", - description="fetch GANA", - descriptionDone="GANA obtained", - command=["./bootstrap" ], - workdir="/home/codespell/sources/exchange/" - ) -) -CODESPELL_FACTORY.addStep( - ShellCommand( - name="doxygen::exchange", - description="checking for doxygen mistakes in exchange", - descriptionDone="doxygen on exchange finished", - command=["/home/codespell/deployment/buildbot/doxygen.sh", "taler.doxy" ], - workdir="/home/codespell/sources/exchange/doc/doxygen" - ) -) -CODESPELL_FACTORY.addStep( - steps.ShellSequence( - name="tag", - description="prepare merchant", - descriptionDone="directory created", - commands=[ - util.ShellArg(command=["mkdir", "-p", "merchant/doc/doxygen/"]), - util.ShellArg(command=["cp", "exchange/doc/doxygen/taler-exchange.tag", "merchant/doc/doxygen/taler-exchange.tag"]), - ], - workdir="/home/codespell/sources/" - ) -) -CODESPELL_FACTORY.addStep( - ShellCommand( - name="doxygen::merchant", - description="checking for doxygen mistakes in merchant", - descriptionDone="doxygen on merchant finished", - command=["/home/codespell/deployment/buildbot/doxygen.sh", "taler.doxy" ], - workdir="/home/codespell/sources/merchant/doc/doxygen" - ) -) - -BUILDERS.append(util.BuilderConfig( - name="codespell-builder", - workernames=["codespell-worker"], - factory=CODESPELL_FACTORY -)) - -EMAIL_ALERTS.append("codespell-builder") - -CODECHANGE_TRIGGERS.append("codespell-builder") - +NIGHTLY_TRIGGERS.append("linkchecker-builder") -################ 11: 'demo checks' JOB ################################### +############################################# +# 19: CONTAINER FACTORY ##################### +############################################# ## -# This worker checks that all the services run under the -# 'demo' deployment are up&running. - -WORKERS.append(worker.Worker("tips-checker-worker", "tips-checker-pass")) -DEMO_CHECK_TIPS_FACTORY = create_factory_with_deployment() -DEMO_CHECK_TIPS_FACTORY.addStep( - ShellCommand( - name="demo tip reserves checker", - description="Checking that demo allocated tip money", - descriptionDone="Demo can tip visitors!.", - command=["./check_tip_reserve.sh"], - workdir="../../deployment/buildbot", +# These factories uses the standard container worker. +WORKERS.append(Worker("container-worker", "container-pass")) + + +# Container Job Generator Functions +# Parse config file and save values in a dict +def ingest_job_config(configPath, jobName): + configDict = {jobName: {}} + print(configDict) + ini.read_string(configPath) + for key in ini["build"]: + value = ini['build'][key] + configDict[jobName][key] = value + print(configDict) + configDict.update(configDict) + print(configDict) + return configDict + + +# Search for configs, and ingest +def handle_job_config(jobDirPath, jobName, repoName, configPath, configExist): + print(configPath) + if configExist == 0: + print(f"Ingesting Job Config: {configPath}") + configDict = ingest_job_config(configPath, jobName) + print(configDict) + return configDict + else: + print("No job config; Using default params") + # Set default job config parameters + configDict = {jobName: {"HALT_ON_FAILURE": True, + "WARN_ON_FAILURE": False, + "CONTAINER_BUILD": True, + "CONTAINER_NAME": repoName, + "CONTAINER_ARCH": "amd64"}} + return configDict + + +class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): + + def __init__(self, REPO_NAME, **kwargs): + self.REPO_NAME = REPO_NAME + kwargs = self.setupShellMixin(kwargs) + super().__init__(**kwargs) + self.observer = logobserver.BufferLogObserver() + self.addLogObserver('stdio', self.observer) + + def extract_stages(self, stdout): + stages = [] + for line in stdout.split('\n'): + stage = str(line.strip()) + if stage: + stages.append(stage) + return stages + + @defer.inlineCallbacks + def run(self): + CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.REPO_NAME}" + CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" + # run 'ls <project_root>/contrib/ci/jobs/' to get the list of stages + cmd = yield self.makeRemoteShellCommand() + yield self.runCommand(cmd) + jobDirs = [] + + # if the command passes extract the list of stages + result = cmd.results() + if result == util.SUCCESS: + jobDirs = self.extract_stages(self.observer.getStdout()) + print(f"this is jobDirs list: {jobDirs}") + self.configDict = {} + print(f"Remote cmd stdout: {self.observer.getStdout()}") + print(f"cmd.results: {cmd.results()}") + for stage in jobDirs: + jobDirPath = f"{CI_JOBS_PATH}/{stage}" + observer = logobserver.BufferLogObserver() + self.addLogObserver('stdio', observer) + cmd1 = yield self.makeRemoteShellCommand( + command=["cat", f"{jobDirPath}/config.ini"]) + yield self.runCommand(cmd1) + print(f"cmd1.results: {cmd1.results()}") + print(f"Second command stdout: {observer.getStdout()}") + print(f"Current stage: {stage}") + print(jobDirPath) + self.configDict.update( + handle_job_config( + jobDirPath, stage, self.REPO_NAME, + observer.getStdout(), cmd1.results())) + print(self.configDict) + # create a container step for each stage and + # add them to the build + convstr2bool = ast.literal_eval + self.build.addStepsAfterCurrentStep([ + container_add_step( + convstr2bool( + str(self.configDict[stage]["HALT_ON_FAILURE"])), + convstr2bool( + str(self.configDict[stage]["WARN_ON_FAILURE"])), + convstr2bool( + str(self.configDict[stage]["CONTAINER_BUILD"])), + self.configDict[stage]["CONTAINER_NAME"], + container_factory, + CONTAINER_WORKDIR, + stage, + self.configDict[stage]["CONTAINER_ARCH"], + f"contrib/ci/jobs/{stage}/job.sh") + for stage in jobDirs + ]) + + return result + + +# List of repos to add to container factory. +container_repos = ["git.taler.net/wallet-core", + "git.taler.net/libeufin", + "git.taler.net/merchant", + "git.taler.net/exchange", + "git.taler.net/docs", + "git.taler.net/taler-ops-www", + "git.taler.net/taler-systems-www", + "git.taler.net/anastasis-www", + "git.taler.net/tutorials", + "git.taler.net/sync", + "git.taler.net/challenger", + "git.taler.net/sandcastle-ng", + "git.gnunet.org/gnunet"] + +for repo in container_repos: + + # Prepare to read job configs + ini = configparser.ConfigParser() + ini.optionxform = str + + # Factory-wide variables + REPO_NAME = repo.rsplit('/', 1)[1] + REPO_URL = "https://" + repo + ".git" + CONTAINER_WORKDIR = f"/home/container-worker/workspace/{REPO_NAME}" + CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" + + # Create a factory + container_factory = util.BuildFactory() + container_factory.workdir = CONTAINER_WORKDIR + + # Setup workspace + container_factory.addStep(ShellCommand( + name="workspace", + descriptionDone="Workspace directory check", + command=f"test -d {CONTAINER_WORKDIR} && podman run --rm --volume {CONTAINER_WORKDIR}:/workdir docker.io/library/debian:bookworm-slim chmod -R 777 /workdir || mkdir -p {CONTAINER_WORKDIR}", haltOnFailure=True, - # Needed to test the 'demo' deployment. - env={"DEPLOYMENT": "demo"} - ) -) + )) -WORKERS.append(worker.Worker("demo-worker", "demo-pass")) -DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment() -DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep( - ShellCommand( - name="demo services checker", - description="Checking demo services are online", - descriptionDone="Demo services are online!.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", + # Ensure repo is cloned or clean. + # Git() will clone repo if it doesn't exist. + # Method clobber removes directory and makes a fresh clone. + # Shallow set to "True" defaults to a depth of 1. + # Will checkout value of "branch" property from job properties. + # https://docs.buildbot.net/latest/manual/configuration/steps/source_git.html + container_factory.addStep(Git( + name="git", + repourl=REPO_URL, + branch=util.Interpolate('%(src::branch)s'), + mode='full', + method='clobber', + shallow=True, + submodules=True, haltOnFailure=True, - # Needed to test the 'demo' deployment. - env={"DEPLOYMENT": "demo"} - ) -) -BUILDERS.append(util.BuilderConfig( - name="demo-services-checker-builder", - workernames="demo-worker", - factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY -)) -BUILDERS.append(util.BuilderConfig( - name="check-tips-builder", - workernames="tips-checker-worker", - factory=DEMO_CHECK_TIPS_FACTORY -)) -EMAIL_ALERTS.append("demo-services-checker-builder") - -# We check demo once per hour. -SCHEDULERS.append(schedulers.Periodic( - name="demo-services-checker-scheduler", - periodicBuildTimer=60 * 60, # 1 hour - builderNames=["demo-services-checker-builder"] -)) - - -################ 12: 'demo health wallet-cli check' JOB ################################### - - -## -# health checks performed by wallet-cli for demo -WORKERS.append(worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass")) - -TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment() -# For the moment, the health-check uses whatever wallet -# was (possibly manually) installed beforehand. This is -# not necessarily a problem, since demo-deployments do never -# update code automatically. -TALER_DEMO_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw-and-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - workdir="../../deployment/buildbot", - command=["./demo-healthchecks.sh"], - ) -) -BUILDERS.append(util.BuilderConfig( - name="taler-demo-healthcheck-builder", - workernames=["taler-demo-healthcheck"], - factory=TALER_DEMO_HEALTHCHECK_FACTORY -)) - -################ 13: 'test health wallet-cli check' JOB ################################### - -## -# health checks performed by wallet-cli for test -WORKERS.append(worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass")) + )) -TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment() -TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="fetch", - description="Running yarn install", - descriptionDone="Correctly installed", - command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"], - workdir="build/" - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-withdraw", - description="Running wallet withdraw tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"], - workdir="build/", - ) -) -TALER_TEST_HEALTHCHECK_FACTORY.addStep( - ShellCommand( - name="test-spend", - description="Running wallet spend tests", - descriptionDone="Test correctly run", - command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"], - workdir="build/", - ) -) -BUILDERS.append(util.BuilderConfig( - name="taler-test-healthcheck-builder", - workernames=["taler-test-healthcheck"], - factory=TALER_TEST_HEALTHCHECK_FACTORY -)) + container_factory.addStep(GenerateStagesCommand( + REPO_NAME, + name="Generate build stages", + command=["ls", CI_JOBS_PATH], + haltOnFailure=True)) + BUILDERS.append(util.BuilderConfig( + name=f"{REPO_NAME}-builder", + workernames=["container-worker"], + factory=container_factory + )) -WALLETCHANGE_TRIGGERS.append("taler-test-healthcheck-builder") + # Only enable this scheduler for debugging! + # Will run builders with 1 minute of waiting inbetween builds + # SCHEDULERS.append(schedulers.Periodic( + # name=f"{REPO_NAME}-minutely", + # builderNames=[f"{REPO_NAME}-builder"], + # periodicBuildTimer=60 + # )) + + SCHEDULERS.append(schedulers.SingleBranchScheduler( + name=f"{REPO_NAME}-container-scheduler", + change_filter=util.ChangeFilter( + branch="master", + project_re=f"({REPO_NAME})" + ), + treeStableTimer=30, + builderNames=[f"{REPO_NAME}-builder"] + )) + SERVICES.append(reporters.MailNotifier( + fromaddr="buildbot@taler.net", + # notify from pass to fail, and viceversa. + generators=[BuildStatusGenerator( + mode=('change','problem','failing','exception',), + builders=[f"{REPO_NAME}-builder",], + message_formatter=reporters.MessageFormatter( + template_type='plain', + wantSteps=True, + wantLogs=True + ), + add_logs=True, + )], + sendToInterestedUsers=False, + useTls=False, + relayhost="localhost", + smtpPort=25, + dumpMailsToLog=True, + extraRecipients=[f"ci-{REPO_NAME}@taler.net"] + )) -################ 14: upgrade test deployment JOB ################################### -## -# testing buildbot using the "buildslavetest" user (for no specific reason except it exists) -# Location: /home/buidlslavetest @ taler.net -WORKERS.append(worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo")) +############## sandcastle-ng Scheduler ################################# -BUILD_FACTORY = create_factory_with_deployment() -BUILD_FACTORY.addStep( - ShellCommand( - name="build", - description="Building all Taler codebase.", - descriptionDone="Taler built.", - command=["./build.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True - ) -) -BUILD_FACTORY.addStep( - ShellCommand( - name="keys generation and sign", - description="Generating exchange keys, and auditor-sign them.", - descriptionDone="Exchange keys generated, and auditor-signed.", - command=["timeout", "--preserve-status", "1m", - "./keys.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'BRANCH': util.Property("branch")} - ) -) -BUILD_FACTORY.addStep( - ShellCommand( - name="restart services", - description="Restarting inactive blue-green party.", - descriptionDone="Restarting Taler.", - command=["./restart.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'BRANCH': util.Property("branch")} - ) -) -BUILD_FACTORY.addStep( - ShellCommand( - name="check services correctly restarted", - description="Checking services are correctly restarted.", - descriptionDone="All services are correctly restarted.", - command=["./checks.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'DEPLOYMENT': "test"} - ) -) -BUILD_FACTORY.addStep( - ShellCommand( - name="create instances", - description="Create merchant instances.", - descriptionDone="All the instances got created.", - command=["./create_instances.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'DEPLOYMENT': "test"} - ) -) - -BUILD_FACTORY.addStep( - ShellCommand( - name="activate tip reserve", - description="Instruct the merchant to pay and authorize the tip reserve.", - descriptionDone="The tip reserve got payed and authorized.", - command=["./create_tip_reserve.sh"], - workdir="../../deployment/buildbot", - haltOnFailure=True, - env={'DEPLOYMENT': "test"} - ) -) - -BUILDERS.append(util.BuilderConfig( - name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY +# Periodic scheduler for sandcastle-ng. +# Runs every 2 hours (60 seconds * 60 * 2) +SCHEDULERS.append(schedulers.Periodic( + name="sandcastle-ng-periodic-scheduler", + builderNames=["sandcastle-ng-builder"], + change_filter=util.ChangeFilter(branch="master"), + periodicBuildTimer=60*60*2 )) -EMAIL_ALERTS.append("test-builder") - -# Scheduler that triggers if anything changes -CODECHANGE_TRIGGERS.append("test-builder") -CODECHANGE_TRIGGERS.append("wallet-builder") - -################ 15: Python linting JOB ################################### - -# This job is noat active / complete yet! -def lint_dispatcher(project): - return "./lint_%s.sh" % project - -LINT_FACTORY = util.BuildFactory() -LINT_FACTORY.addStep( - ShellCommand( - name="Python linter", - description="linting Python", - descriptionDone="linting done", - command=util.Transform(lint_dispatcher, util.Property("project")), - workdir="../../deployment/taler-build" - ) -) -# This builder is NOT ACTIVE! -#BUILDERS.append(util.BuilderConfig( -# name="lint-builder", -# workernames=["lint-worker"], -# factory=LINT_FACTORY -#)) - -# Consider adding other Python parts, like the various frontends. -# NOTE: scheduler is NOT active! (commented out below) -#SCHEDULERS.append(schedulers.SingleBranchScheduler( -# name="lint-scheduler", -# change_filter=util.ChangeFilter( -# branch="master", project_re="(bank|donations|survey|blog)" -# ), -# treeStableTimer=None, -# builderNames=["lint-builder"] -#)) - - -################ 16: Selenium JOB ################################### - -# This job is noat active! -SELENIUM_FACTORY = create_factory_with_deployment() -SELENIUM_FACTORY.addStep( - ShellCommand( - name="selenium", - description="Headless browser test", - descriptionDone="Test finished", - command=["launch_selenium_test"], - env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"} - ) -) - -#BUILDERS.append(util.BuilderConfig( -# name="selenium-builder", -# workernames=["selenium-worker"], -# factory=SELENIUM_FACTORY -#)) - ################ 99: debug stuff JOB ################################### @@ -1147,21 +672,6 @@ BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) ####### GENERAL PURPOSE BUILDBOT SERVICES ####################### -SERVICES.append(reporters.IRC( - "irc.eu.libera.net", - "taler-bb", - useColors=False, - channels=[{ - "channel": "#taler" - }], - password="taler-bb-pass19", - notify_events={ - 'exception': 1, - 'successToFailure': 1, - 'failureToSuccess': 1 - } -)) - SERVICES.append(reporters.MailNotifier( fromaddr="testbuild@taler.net", # notify from pass to fail, and viceversa. @@ -1176,7 +686,11 @@ SERVICES.append(reporters.MailNotifier( add_logs=True, )], sendToInterestedUsers=False, - extraRecipients=["buildfailures@taler.net"] + useTls=False, + relayhost="localhost", + smtpPort=25, + dumpMailsToLog=True, + extraRecipients=BUILDER_EMAIL_ADDRESSES )) @@ -1248,7 +762,16 @@ c["db"] = { # the 'change_source' setting tells the buildmaster how it should # find out about source code changes. -c["change_source"] = [changes.PBChangeSource(user="allcs", passwd="allcs")] +pbSource = PBChangeSource(user="allcs", passwd="allcs") + + +pollGnunetSource = changes.GitPoller(repourl='https://git.gnunet.org/gnunet.git', + branches=True, + pollInterval=300, + pollAtLaunch=True, + project="gnunet") + +c["change_source"] = [pollGnunetSource, pbSource] # 'protocols' contains information about protocols which master # will use for communicating with workers. You must define at @@ -1265,7 +788,7 @@ c["buildbotURL"] = "https://buildbot.taler.net/" # -- formerly commented out as not packaged properly in Debian and others, see # https://bugzilla.redhat.com/show_bug.cgi?id=1557687 c["www"] = { - "port": 8010, + "port": "tcp:8010:interface=127.0.0.1", "default_page": 'waterfall', "plugins": { "waterfall_view": True, |