path: root/buildbot/master.cfg

##
# This file is part of TALER
# (C) 2016-2021 Taler Systems SA
#
# TALER is free software; you can redistribute it and/or
# modify it under the terms of the GNU Affero General Public
# License as published by the Free Software Foundation; either
# version 3, or (at your option) any later version.
#
# TALER is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty
# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with TALER; see the file COPYING.  If not,
# see <http://www.gnu.org/licenses/>
#
# @author Florian Dold
# @author Marcello Stanisci
# @author ng0
# @author Christian Grothoff
import re
from getpass import getuser
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.plugins import *
from buildbot.reporters.generators.build import BuildStatusGenerator

# This is a sample buildmaster config file. It must be
# installed as 'master.cfg' in your buildmaster's base
# directory.

# This file has the following structure:
# - Globals: definition of global variables we use throughout
#   + Convenience functions: helper functions useful for many jobs
# - Jobs: actual job definitions
# - General purpose: triggers and alerts shared by various jobs
#   + general purpose notification (alerts)
#   + general purpose triggers (schedulers)
# - Actual buildbot configuration object initialization

#################################################################
######################### GLOBALS ###############################
#################################################################

# The 'workers' list defines the set of recognized workers.
# Each element is a Worker object, specifying a unique worker
# name and password.  The same worker name and password must
# be configured on the worker.
WORKERS = []

# 'services' is a list of BuildbotService items like reporter
# targets. The status of each build will be pushed to these
# targets. buildbot/reporters/*.py has a variety to choose from,
# like IRC bots.
SERVICES = []

# The 'builders' list defines the Builders, which tell Buildbot
# how to perform a build: what steps, and which workers can execute
# them.  Note that any particular build will only take place on
# one worker.
BUILDERS = []

# Configures the Schedulers, which decide how to react to incoming
# changes.
SCHEDULERS = []

# Array of builders to be scheduled every night.
NIGHTLY_TRIGGERS=[]

# Array of builders to be scheduled whenever any of the code Git repos change
CODECHANGE_TRIGGERS=[]

# Array of builders to be scheduled whenever the wallet-core or deployment change
WALLETCHANGE_TRIGGERS=[]

# Array of builder names for which build status reports should be sent via e-mail
EMAIL_ALERTS=[]


############ Convenience functions #################

# Create a FACTORY with a deployment.git checkout as the first step.
def create_factory_with_deployment():
    f = util.BuildFactory()
    update_deployment (f)
    return f

# Convenience function that checks out a Git repository.
# First argument is the URL of the Git to clone, second
# the desired branch. Default is 'master'.
def git_step(repo,target_branch="master"):
    return Git(
        repourl=repo,
        mode="full",
        method="fresh",
        logEnviron=False,
        alwaysUseLatest=True,
        haltOnFailure=True,
        branch=target_branch
    )

# Convenience function that runs 'make check' in a
# directory of the code inside of a netjail.
def jailed_check(package,srcdirs):
    return steps.ShellSequence(
        name="Tests of " + package,
        description="Testing " + package,
        descriptionDone="Pass",
        commands=map(lambda srcdir: util.ShellArg(command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "bash", "-c", "'cd src/"+srcdir+" make check'"]), srcdirs),
        workdir="../../sources/" + package
    )

# Convenience function that checks out the deployment.
def update_deployment(factory):
    factory.addStep(steps.ShellSequence(
        name="update deployment",
        description="removing old deployment and fetching fresh repository",
        descriptionDone="Deployment updated",
        commands=[
            util.ShellArg(command=["rm", "-rf", "deployment"]),
            util.ShellArg(command=["git", "clone", "git://git.taler.net/deployment"]),
        ],
        haltOnFailure=True,
        workdir="../.."
    ))


##################################################################
######################## JOBS ####################################
##################################################################

# For every job, we have (in this order!):
# - worker(s): hosts/users that run the job
# - factory: list of steps that define what to do
# - builder: gives the job a name and binds it to the factory and worker
# - (OPTIONAL) alerts: notifications to trigger when the job fails
#   Pre-defined: EMAIL_ALERTS
# - scheduler: rules that define when to run the job
#   Pre-defined: NIGHTLY_TRIGGERS, CODECHANGE_TRIGGERS, WALLETCHANGE_TRIGGERS

################ 1: BUILDMASTER JOB ###################################

##
# This worker restarts the buildmaster itself on
# changes to this file.
# Location: /home/buildbot-master @ taler.net
WORKERS.append(worker.Worker("buildmaster-worker", "buildmaster-pass"))

BUILDMASTER_FACTORY = create_factory_with_deployment()
BUILDMASTER_FACTORY.addStep(
    ShellCommand(
        name="restart buildmaster",
        description="trigger buildmaster restart with new configuration",
        descriptionDone="Buildmaster updated",
        command=["systemctl", "--user", "kill", "--signal", "SIGHUP", "buildbot-master"],
        workdir="../.."
    )
)

BUILDERS.append(util.BuilderConfig(
    name="buildmaster-builder",
    workernames=["buildmaster-worker"],
    factory=BUILDMASTER_FACTORY
))

EMAIL_ALERTS.append("buildmaster-builder")

# Buildmaster is notified whenever deployment.git changes
SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="buildmaster-scheduler",
    change_filter=util.ChangeFilter(
        branch="master",
        project_re="(deployment)"
    ),
    treeStableTimer=None,
    builderNames=["buildmaster-builder"]
))


################ 2: DOCUMENTATION JOB ###################################

##
# This worker builds manuals / API docs / tutorials.
# Location: /home/docbuilder @ taler.net
WORKERS.append(worker.Worker("doc-worker", "doc-pass"))

DOC_FACTORY = create_factory_with_deployment()
DOC_FACTORY.addStep(
    ShellCommand(
        name="build docs",
        description="Building documentation",
        descriptionDone="Documentation built.",
        command=["./build-docs.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)
DOC_FACTORY.addStep(
    steps.ShellSequence(
        name="prepare exchange",
        description="Running bootstrap and configure for exchange",
        descriptionDone="exchange ready for doxygen",
        commands=[
             util.ShellArg(command=["./bootstrap"], logname='bootstrap'),
             util.ShellArg(command=["./configure", "--enable-only-doc"], logname='configure'),
        ],
        workdir="../../sources/exchange/",
        haltOnFailure=True,
    )
)
DOC_FACTORY.addStep(
    ShellCommand(
        name="doxygen::exchange",
        description="building exchange doxygen documentation",
        descriptionDone="doxygen on exchange finished",
        command=["make", "full" ],
        want_stderr=False,
        workdir="../../sources/exchange/doc/doxygen",
        haltOnFailure=True,
    )
)
DOC_FACTORY.addStep(
    steps.ShellSequence(
        name="prepare merchant",
        description="prepare merchant",
        descriptionDone="merchant prepared",
        commands=[
            util.ShellArg(command=["./bootstrap"], logname='bootstrap'),
            util.ShellArg(command=["./configure", "--enable-only-doc"], logname='configure'),
            util.ShellArg(command=["cp", "../exchange/doc/doxygen/taler-exchange.tag", "doc/doxygen/taler-exchange.tag"]),
        ],
        workdir="../../sources/merchant/"
    )
)
DOC_FACTORY.addStep(
    ShellCommand(
        name="doxygen::merchant",
        description="building merchant doxygen documentation",
        descriptionDone="doxygen on merchant finished",
        command=["make", "full" ],
        want_stderr=False,
        workdir="../../sources/merchant/doc/doxygen"
    )
)
DOC_FACTORY.addStep(
    steps.ShellSequence(
        name="prepare anastasis",
        description="prepare anastasis",
        descriptionDone="doxygen on anastasis finished",
        commands=[
            util.ShellArg(command=["./bootstrap"], logname='bootstrap'),
            util.ShellArg(command=["./configure", "--enable-only-doc"], logname='configure'),
            util.ShellArg(command=["cp", "../exchange/doc/doxygen/taler-exchange.tag", "doc/doxygen/taler-exchange.tag"], logname="cp-e"),
            util.ShellArg(command=["cp", "../merchant/doc/doxygen/taler-merchant.tag", "doc/doxygen/taler-merchant.tag"], logname="cp-m"),
        ],
        workdir="../../sources/anastasis/"
    )
)
DOC_FACTORY.addStep(
    ShellCommand(
        name="doxygen::anastasis",
        description="building anastasis doxygen documentation",
        descriptionDone="doxygen on anastasis finished",
        command=["make", "full" ],
        want_stderr=False,
        workdir="../../sources/anastasis/doc/doxygen"
    )
)
DOC_FACTORY.addStep(
    steps.ShellSequence(
        name="doxygen::wallet",
        description="building wallet typescript documentation",
        descriptionDone="typedoc on taler-wallet-core finished",
        commands=[
            util.ShellArg(command=["./bootstrap"], logname="bootstrap"),
            util.ShellArg(command=["./configure"], logname="configure"),
            util.ShellArg(command=["make"], logname="make"),
            util.ShellArg(command=["pnpm", "install", "-W", "typedoc"], logname="pnpm"),
            util.ShellArg(command=["./node_modules/typedoc/bin/typedoc", "--out", "dist/typedoc", "--tsconfig", "tsconfig.build.json", "packages/taler-util/src/index.ts", "packages/taler-wallet-cli/src/index.ts", "packages/taler-wallet-android/src/index.ts", "packages/taler-wallet-core/src/index.ts" ], logname="typedoc"),
        ],
        workdir="../../sources/wallet-core/"
    )
)


BUILDERS.append(util.BuilderConfig(
    name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY
))

EMAIL_ALERTS.append("doc-builder")

#sphinxErrorNotifier = reporters.MailNotifier(
#    fromaddr="bb@taler.net",
#    sendToInterestedUsers=False,
#    addLogs=['build docs.stdio',],
#    useTls=True,
#    # notify if sphinx exits with error (command line option in Makefile turns warnings into exit 1)
#    mode=('failing'),
#    builders=('doc-builder',),
#    extraRecipients=['sphinxerrors@taler.net']
#)


# Docs run if master or stable branch of 'docs' (or deployment) changed.
SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="periodic-doc-scheduler",
    builderNames=["doc-builder"],
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)", project_re="(docs|deployment)"
    ),
    treeStableTimer=None,
))


################ 3: WEBSITE JOB ###################################

##
# This worker builds Websites: www and stage.
#
WORKERS.append(worker.Worker("sites-worker", "sites-pass"))

SITES_FACTORY = create_factory_with_deployment()
SITES_FACTORY.addStep(
    ShellCommand(
        name="build Web sites",
        description="Building all the Taler homepages",
        descriptionDone="Sites built.",
        command=["./build-sites.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)

BUILDERS.append(util.BuilderConfig(
    name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY
))

EMAIL_ALERTS.append("sites-builder")

# The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed:
def twister_web_page(change):
    _change = change.asDict()
    repo = _change.get("project")
    if repo in ["www", "buywith", "deployment"]:
        return True

    files = _change.get("files")
    for file in files:
        if re.search(r"web", file.get("name", "")) \
                and "twister" == repo:
            return True
    return False

# Sites are re-build whenever deployment, www buywith, or twister changes.
SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="sites-scheduler",
    builderNames=["sites-builder"],
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)",
        filter_fn=twister_web_page
    ),
    treeStableTimer=None
))


################ 4: LCOV JOB ###################################

##
# This worker makes the code coverage and publishes it
# under the "lcov" Website.
WORKERS.append(worker.Worker("lcov-worker", "lcov-pass"))

LCOV_FACTORY = create_factory_with_deployment()
LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
LCOV_FACTORY.addStep(
    ShellCommand(
        name="fetch",
        description="Running yarn install of wallet",
        descriptionDone="Correctly installed",
        command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"],
        workdir="build/",
        haltOnFailure=True,
    )
)
LCOV_FACTORY.addStep(
    ShellCommand(
        name="build",
        description="Building other Taler components",
        descriptionDone="Taler built",
        command=["./build.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'TALER_COVERAGE': "1"},
    )
)
LCOV_FACTORY.addStep(
    ShellCommand(
        name="coverage generation",
        description="running tests",
        descriptionDone="generating HTML report",
        command=["/usr/bin/sudo", "/usr/local/bin/netjail.sh", "/home/lcovworker/deployment/buildbot/with-postgres.sh","./coverage.sh"],
        workdir="../../deployment/buildbot",
        env={'PATH': "${HOME}/local/bin:${PATH}"}
    )
)

BUILDERS.append(util.BuilderConfig(
    name="lcov-builder",
    workernames=["lcov-worker"],
    factory=LCOV_FACTORY
))

EMAIL_ALERTS.append("lcov-builder")
NIGHTLY_TRIGGERS.append("lcov-builder")


################ 5: UNIT TEST JOB ###################################

##
# This worker builds everything and runs our 'make check'
# test suite against 'everything'.
WORKERS.append(worker.Worker("checker-worker", "checker-pass"))

INTEGRATIONTEST_FACTORY = create_factory_with_deployment()
INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="fetch gnunet.org sources",
        description="fetching latest deployment repositories from git.gnunet.org",
        descriptionDone="GNUnet code base updated",
        command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
    )
)
INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="fetch taler.net sources",
        description="fetching latest deployment repositories from git.taler.net",
        descriptionDone="Taler code base updated",
        command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "anastasis", "bank", "twister"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
    )
)
INTEGRATIONTEST_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="fetch",
        description="Running yarn install of wallet",
        descriptionDone="Correctly installed",
        command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"],
        workdir="../../sources/wallet-core",
        haltOnFailure=True,
    )
)
INTEGRATIONTEST_FACTORY.addStep(
    steps.ShellSequence(
        name="fetch",
        description="Running local install of wallet",
        descriptionDone="local wallet install done",
        commands=[
            util.ShellArg(command=["./bootstrap"]),
            util.ShellArg(command=["./configure", "--prefix=$HOME/local/"]),
            util.ShellArg(command=["make"]),
        ],
        workdir="../../sources/wallet-core",
        haltOnFailure=True,
    )
)
INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="build",
        description="Building other Taler components",
        descriptionDone="Taler built",
        command=["./build.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=False
    )
)
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", [
    "microhttpd",
    "testcurl",
    "testzzuf",
]))
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", [
    "util",
    "pq",
    "curl",
    "json",
]))
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("twister", [
    "test",
]))
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", [
   "util",
   "curl",
   "mhd",
   "pq",
   "json",
   "bank-lib",
   "exchangedb",
   "auditordb",
   "exchange",
   "auditor",
   "lib",
   "exchange-tools",
   "testing",
   "benchmark",
]))
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", [
   "mustach",
   "backenddb",
   "backend",
   "lib",
   "testing",
   "merchant-tools",
]))
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", [
    "util",
    "syncdb",
    "sync",
    "lib"
]))
INTEGRATIONTEST_FACTORY.addStep(jailed_check ("anastasis", [
    "util",
    "stasis",
    "testing",
    "reducer",
]))
INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="bank check",
        description="Testing Taler Python bank",
        descriptionDone="Done",
        command=["sudo", "/usr/local/bin/netjail.sh", "/home/integrationtest/deployment/buildbot/with-postgres.sh", "make", "check"],
        workdir="../../sources/bank",
        haltOnFailure=False,
        env={'PYTHONUSERBASE': "$HOME/local" }
    )
)
INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="wallet check",
        description="Testing wallet-core",
        descriptionDone="Done",
        command=["make", "check"],
        workdir="../../sources/wallet-core",
        haltOnFailure=False
    )
)

BUILDERS.append(util.BuilderConfig(
    name="checker-builder",
    workernames=["checker-worker"],
    factory=INTEGRATIONTEST_FACTORY
))

EMAIL_ALERTS.append("checker-builder")
CODECHANGE_TRIGGERS.append("checker-builder")


################ 6: 'test.taler.net' deployment JOB ###################################

##
# This worker builds Taler for the 'test' deployment.
WORKERS.append(worker.Worker("test-worker", "test-pass"))

# buildslavetest FACTORY
BUILDSLAVETEST_FACTORY = create_factory_with_deployment()
BUILDSLAVETEST_FACTORY.addStep(
    ShellCommand(
        name="buildslavetest script (for testing purposes)",
        description="Build Slave Test",
        descriptionDone="buildslavetest: Done",
        command=["./buildslavetest.sh"],
        workdir="/home/buildslavetest/"
    )
)

# buildslavetest BUILDER
BUILDERS.append(util.BuilderConfig(
    name="buildslavetest-builder",
    workernames=["buildslavetest-worker"],
    factory=BUILDSLAVETEST_FACTORY
))

EMAIL_ALERTS.append("buildslavetest-builder")

# buildslavetest SCHEDULER
SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="buildslavetest-scheduler",
    builderNames=["buildslavetest-builder"],
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)", project_re="(help|deployment)"
    ),
    treeStableTimer=None,
))

################ 7: 'test-auditor' deployment JOB ###################################


##
# This worker compiles the auditor reports for the "test"
# demo deployment.
WORKERS.append(worker.Worker("test-auditor-worker", "test-auditor-pass"))

AUDITOR_FACTORY_TEST = create_factory_with_deployment()
AUDITOR_FACTORY_TEST.addStep(
    ShellCommand(
        name="Auditor reports generator",
        description="Generating auditor reports.",
        descriptionDone="Auditor reports correctly generated.",
        command=["./make_auditor_reports.sh"],
        workdir="../../deployment/buildbot",
        env={'TALER_HOME': "/home/taler-test/"}
   )
)

BUILDERS.append(util.BuilderConfig(
    name="auditor-builder-test",
    workernames=["test-auditor-worker"],
    factory=AUDITOR_FACTORY_TEST
))
NIGHTLY_TRIGGERS.append("auditor-builder-test")


################ 8: 'demo-auditor' deployment JOB ###################################
##
# This worker compiles the auditor reports for the "green"
# demo deployment.
WORKERS.append(worker.Worker("demo-auditor-worker", "demo-auditor-pass"))

AUDITOR_FACTORY_DEMO = create_factory_with_deployment()
AUDITOR_FACTORY_DEMO.addStep(
    ShellCommand(
        name="Auditor reports generator",
        description="Generating auditor reports.",
        descriptionDone="Auditor reports correctly generated.",
        command=["./make_auditor_reports.sh"],
        workdir="../../deployment/buildbot",
        env={'TALER_HOME': "/home/demo/active-home/"}
   )
)

BUILDERS.append(util.BuilderConfig(
    name="auditor-builder-demo",
    workernames=["demo-auditor-worker"],
    factory=AUDITOR_FACTORY_DEMO
))

NIGHTLY_TRIGGERS.append("auditor-builder-demo")


################ 8: 'build wallet-core' JOB ###################################

##
# This worker builds wallet-core.
WORKERS.append(worker.Worker("wallet-worker", "wallet-pass"))

WALLET_FACTORY = create_factory_with_deployment()
WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
WALLET_FACTORY.addStep(
    ShellCommand(
        name="fetch",
        description="Running yarn install",
        descriptionDone="Correctly installed",
        command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"],
        workdir="build/"
    )
)
WALLET_FACTORY.addStep(
    ShellCommand(
        name="test",
        description="Running wallet tests",
        descriptionDone="Test correctly run",
        command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "testing", "run-integrationtests"],
        workdir="build/",
    )
)
# WALLET_FACTORY.addStep(
#     ShellCommand(
#         name="lint",
#         description="Linting the wallet",
#         descriptionDone="Linting done",
#         command=["make", "lint"],
#         workdir="build/"
#     )
# )

BUILDERS.append(util.BuilderConfig(
    name="wallet-builder",
    workernames=["wallet-worker"],
    factory=WALLET_FACTORY
))

EMAIL_ALERTS.append("wallet-builder")

# Wallet is re-build whenever wallet-core or deployment changes
SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="wallet-scheduler",
    change_filter=util.ChangeFilter(
        branch="master",
        project_re="(wallet-core|deployment)"
    ),
    treeStableTimer=None,
    builderNames=["wallet-builder"]
))


################ 9: 'check links' JOB ###################################

##
# linkchecker worker checks for dead links in the Website
# Location: /home/linkchecker @ taler.net
WORKERS.append(worker.Worker("linkchecker-worker", "linkchecker-pass"))

# linkchecker FACTORY
LINKCHECKER_FACTORY = create_factory_with_deployment()
LINKCHECKER_FACTORY.addStep(
    ShellCommand(
        name="linkchecker",
        description="Check taler.net website for broken links && Notify",
        descriptionDone="Results of wget in buildbot logs.",
        command=["/home/linkchecker/deployment/buildbot/linkchecker.sh"],
        workdir="/home/linkchecker",
        haltOnFailure=True,
    )
)

# linkchecker BUILDER
# worker at linkchecker@taler.net
BUILDERS.append(util.BuilderConfig(
    name="linkchecker-builder",
    workernames="linkchecker-worker",
    factory=LINKCHECKER_FACTORY
))

docs_generator = BuildStatusGenerator(
    mode=('change','problem','failing','exception',),
    builders=[
       'linkchecker-builder',
    ],
    message_formatter=reporters.MessageFormatter(
       template_type='plain',
       wantSteps=True,
       wantLogs=True
     ),
     add_logs=True
    )

SERVICES.append(reporters.MailNotifier(
    fromaddr="bb@taler.net",
    generators=[docs_generator],
    sendToInterestedUsers=False,
    useTls=True,
    extraRecipients=['linkcheck@taler.net']
))

NIGHTLY_TRIGGERS.append("linkchecker-builder")


################ 10: 'check spelling' JOB ###################################

##
# codespell worker checks for spelling mistakes in code
# Location: /home/codespell @ taler.net
WORKERS.append(worker.Worker("codespell-worker", "codespell-pass"))

CODESPELL_FACTORY = create_factory_with_deployment()
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="clean old deployment",
        description="cleaning previous doxygen runs",
        descriptionDone="Doxygen cleaned",
        command=["rm", "-rf", "exchange/doc/doxygen/html/", "merchant/doc/doxygen/html/" ],
        workdir="/home/codespell/sources/"
    )
)
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="fetch gnunet.org sources",
        description="fetching latest deployment repositories from git.gnunet.org",
        descriptionDone="GNUnet code base updated",
        command=["./update-sources.sh", "git://git.gnunet.org/", "libmicrohttpd", "gnunet"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
    )
)
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="fetch taler.net sources",
        description="fetching latest deployment repositories from git.taler.net",
        descriptionDone="Taler code base updated",
        command=["./update-sources.sh", "git://git.taler.net/", "exchange", "merchant", "wallet-core", "sync", "anastasis", "bank", "twister"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
    )
)
CODESPELL_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="codespell",
        description="spell checking repositories",
        descriptionDone="Spell check complete",
        command=["/home/codespell/.local/bin/codespell", "-I", "/home/codespell/deployment/codespell/dictionary.txt", "-S", "*.bib,*.bst,*.cls,*.png,*.svg,*.wav,*.gz,*/mustach/**,*.fees,*key,*.tag,*.info,*.latexmkrc,*.ecc,*.jpg,*.zkey,*.sqlite,*/contrib/hellos/**,*/vpn/tests/**,*.priv,*.file,*.tgz,*.woff,*.gif,*.odt,*.fee,*.deflate,*.dat,*.jpeg,*.eps,*.odg,*/m4/ax_lib_postgresql.m4,*/m4/libgcrypt.m4,*.rpath,config.status,ABOUT-NLS,*/doc/texinfo.tex,*.PNG,*.??.json,*.docx,*.ods,*.doc,*.docx,*.xcf,*.xlsx,*.ecc,*.ttf,*.woff2,*.eot,*.ttf,*.eot,*.mp4,*.pptx,*.epgz,*.min.js,*.pack.js,*.po,*.bbl,*/afl-tests/*,*/.git/**,*.pdf,*.epub,**/signing-key.asc,**/pnpm-lock.yaml,**/*.svg,**/*.cls,**/rfc.bib,**/*.bst", "anastasis", "bank", "exchange", "gnunet", "libmicrohttpd", "merchant", "sync", "twister", "wallet-core"],
        workdir="/home/codespell/sources/"
    )
)
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="gana",
        description="fetch GANA",
        descriptionDone="GANA obtained",
        command=["contrib/gana.sh" ],
        workdir="/home/codespell/sources/exchange/"
    )
)
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="doxygen::exchange",
        description="checking for doxygen mistakes in exchange",
        descriptionDone="doxygen on exchange finished",
        command=["/home/codespell/deployment/buildbot/doxygen.sh", "taler.doxy" ],
        workdir="/home/codespell/sources/exchange/doc/doxygen"
    )
)
CODESPELL_FACTORY.addStep(
    steps.ShellSequence(
        name="tag",
        description="prepare merchant",
        descriptionDone="directory created",
        commands=[
            util.ShellArg(command=["mkdir", "-p", "merchant/doc/doxygen/"]),
            util.ShellArg(command=["cp", "exchange/doc/doxygen/taler-exchange.tag", "merchant/doc/doxygen/taler-exchange.tag"]),
        ],
        workdir="/home/codespell/sources/"
    )
)
CODESPELL_FACTORY.addStep(
    ShellCommand(
        name="doxygen::merchant",
        description="checking for doxygen mistakes in merchant",
        descriptionDone="doxygen on merchant finished",
        command=["/home/codespell/deployment/buildbot/doxygen.sh", "taler.doxy" ],
        workdir="/home/codespell/sources/merchant/doc/doxygen"
    )
)

BUILDERS.append(util.BuilderConfig(
    name="codespell-builder",
    workernames=["codespell-worker"],
    factory=CODESPELL_FACTORY
))

EMAIL_ALERTS.append("codespell-builder")

CODECHANGE_TRIGGERS.append("codespell-builder")


################ 11: 'demo checks' JOB ###################################

##
# This worker checks that all the services run under the
# 'demo' deployment are up&running.
WORKERS.append(worker.Worker("demo-worker", "demo-pass"))

DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment()
DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep(
    ShellCommand(
        name="demo services checker",
        description="Checking demo services are online",
        descriptionDone="Demo services are online!.",
        command=["./checks.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        # Needed to test the 'demo' deployment.
        env={"DEPLOYMENT": "demo"}
    )
)
BUILDERS.append(util.BuilderConfig(
    name="demo-services-checker-builder",
    workernames="demo-worker",
    factory=DEMO_SERVICES_INTEGRATIONTEST_FACTORY
))
EMAIL_ALERTS.append("demo-services-checker-builder")

# We check demo once per hour.
SCHEDULERS.append(schedulers.Periodic(
    name="demo-services-checker-scheduler",
    periodicBuildTimer=60 * 60,  # 1 hour
    builderNames=["demo-services-checker-builder"]
))


################ 12: 'demo health wallet-cli check' JOB ###################################


##
# health checks performed by wallet-cli for demo
WORKERS.append(worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass"))

TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment()
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
    ShellCommand(
        name="fetch",
        description="Running yarn install",
        descriptionDone="Correctly installed",
        command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"],
        workdir="build/"
    )
)
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
    ShellCommand(
        name="test-withdraw",
        description="Running wallet withdraw tests",
        descriptionDone="Test correctly run",
        command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "testing", "run-integrationtest", "-b", "https://bank.demo.taler.net", "-w", "TESTKUDOS:10"],
        workdir="build/",
    )
)
TALER_DEMO_HEALTHCHECK_FACTORY.addStep(
    ShellCommand(
        name="test-spend",
        description="Running wallet spend tests",
        descriptionDone="Test correctly run",
        command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.demo.taler.net", "-s", "TESTKUDOS:4"],
        workdir="build/",
    )
)
BUILDERS.append(util.BuilderConfig(
    name="taler-demo-healthcheck-builder",
    workernames=["taler-demo-healthcheck"],
    factory=TALER_DEMO_HEALTHCHECK_FACTORY
))

WALLETCHANGE_TRIGGERS.append("taler-demo-healthcheck-builder")

################ 13: 'test health wallet-cli check' JOB ###################################

##
# health checks performed by wallet-cli for test
WORKERS.append(worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass"))

TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment()
TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
TALER_TEST_HEALTHCHECK_FACTORY.addStep(
    ShellCommand(
        name="fetch",
        description="Running yarn install",
        descriptionDone="Correctly installed",
        command=["npm", "install", "-g", "--prefix", "$HOME", "@gnu-taler/taler-wallet-cli"],
        workdir="build/"
    )
)
TALER_TEST_HEALTHCHECK_FACTORY.addStep(
    ShellCommand(
        name="test-withdraw",
        description="Running wallet withdraw tests",
        descriptionDone="Test correctly run",
        command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-w", "TESTKUDOS:10"],
        workdir="build/",
    )
)
TALER_TEST_HEALTHCHECK_FACTORY.addStep(
    ShellCommand(
        name="test-spend",
        description="Running wallet spend tests",
        descriptionDone="Test correctly run",
        command=["timeout", "--preserve-status", "5m", "./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose", "-b", "https://bank.test.taler.net", "-s", "TESTKUDOS:4"],
        workdir="build/",
    )
)
BUILDERS.append(util.BuilderConfig(
    name="taler-test-healthcheck-builder",
    workernames=["taler-test-healthcheck"],
    factory=TALER_TEST_HEALTHCHECK_FACTORY
))


WALLETCHANGE_TRIGGERS.append("taler-test-healthcheck-builder")


################ 14: upgrade test deployment JOB ###################################

##
# testing buildbot using the "buildslavetest" user (for no specific reason except it exists)
# Location: /home/buidlslavetest @ taler.net
WORKERS.append(worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo"))

BUILD_FACTORY = create_factory_with_deployment()
BUILD_FACTORY.addStep(
    ShellCommand(
        name="build",
        description="Building all Taler codebase.",
        descriptionDone="Taler built.",
        command=["./build.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)
BUILD_FACTORY.addStep(
    ShellCommand(
        name="config",
        description="Generating configuration file.",
        descriptionDone="Configuration file generated.",
        command=["./config.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)
BUILD_FACTORY.addStep(
    ShellCommand(
        name="keys generation and sign",
        description="Generating exchange keys, and auditor-sign them.",
        descriptionDone="Exchange keys generated, and auditor-signed.",
        command=["./keys.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'BRANCH': util.Property("branch")}
    )
)
BUILD_FACTORY.addStep(
    ShellCommand(
        name="restart services",
        description="Restarting inactive blue-green party.",
        descriptionDone="Restarting Taler.",
        command=["./restart.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'BRANCH': util.Property("branch")}
    )
)
BUILD_FACTORY.addStep(
    ShellCommand(
        name="check services correctly restarted",
        description="Checking services are correctly restarted.",
        descriptionDone="All services are correctly restarted.",
        command=["./checks.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'DEPLOYMENT': "test"}
    )
)
BUILD_FACTORY.addStep(
    ShellCommand(
        name="create instances",
        description="Create merchant instances.",
        descriptionDone="All the instances got created.",
        command=["./create_instances.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'DEPLOYMENT': "test"}
    )
)

BUILDERS.append(util.BuilderConfig(
    name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY
))

EMAIL_ALERTS.append("test-builder")

# Scheduler that triggers if anything changes
CODECHANGE_TRIGGERS.append("test-builder")


################ 15: Python linting JOB ###################################

# This job is noat active / complete yet!
def lint_dispatcher(project):
    return "./lint_%s.sh" % project

LINT_FACTORY = util.BuildFactory()
LINT_FACTORY.addStep(
    ShellCommand(
        name="Python linter",
        description="linting Python",
        descriptionDone="linting done",
        command=util.Transform(lint_dispatcher, util.Property("project")),
        workdir="../../deployment/taler-build"
    )
)
# This builder is NOT ACTIVE!
#BUILDERS.append(util.BuilderConfig(
#    name="lint-builder",
#    workernames=["lint-worker"],
#    factory=LINT_FACTORY
#))

# Consider adding other Python parts, like the various frontends.
# NOTE: scheduler is NOT active! (commented out below)
#SCHEDULERS.append(schedulers.SingleBranchScheduler(
#    name="lint-scheduler",
#    change_filter=util.ChangeFilter(
#        branch="master", project_re="(bank|donations|survey|blog)"
#    ),
#    treeStableTimer=None,
#    builderNames=["lint-builder"]
#))


################ 16: Selenium JOB ###################################

# This job is noat active!
SELENIUM_FACTORY = create_factory_with_deployment()
SELENIUM_FACTORY.addStep(
    ShellCommand(
        name="selenium",
        description="Headless browser test",
        descriptionDone="Test finished",
        command=["launch_selenium_test"],
        env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"}
    )
)

#BUILDERS.append(util.BuilderConfig(
#    name="selenium-builder",
#    workernames=["selenium-worker"],
#    factory=SELENIUM_FACTORY
#))


################ 99: debug stuff JOB ###################################

# This does nothing, just a starting point for a factory.
DEBUG_FACTORY = util.BuildFactory()
DEBUG_FACTORY.addStep(
    ShellCommand(
        name="echo debug",
        description="just echoing a word",
        descriptionDone="builder responded",
        command=["echo", "I'm here!"]
    )
)


##################################################################
#################### General purpose #############################
##################################################################

# Compute array of the names of all of our builders
BUILDER_LIST = map(lambda builder: builder.name, BUILDERS)

####### GENERAL PURPOSE BUILDBOT SERVICES #######################

SERVICES.append(reporters.IRC(
    "irc.eu.freenode.net",
    "taler-bb",
    useColors=False,
    channels=[{
        "channel": "#taler"
    }],
    password="taler-bb-pass19",
    notify_events={
        'exception': 1,
        'successToFailure': 1,
        'failureToSuccess': 1
    }
))

SERVICES.append(reporters.MailNotifier(
    fromaddr="testbuild@taler.net",
    # notify from pass to fail, and viceversa.
    generators=[BuildStatusGenerator(
        mode=('change','problem','failing','exception',),
        builders=EMAIL_ALERTS,
        message_formatter=reporters.MessageFormatter(
            template_type='plain',
            wantSteps=True,
            wantLogs=True
        ),
        add_logs=True,
    )],
    sendToInterestedUsers=False,
    extraRecipients=["buildfailures@taler.net"]
))


############# GENERAL PURPOSE SCHEDULERS ##########################

# Workers that are done on wallet or deployment changes to master
SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="taler-healthcheck-scheduler",
    change_filter=util.ChangeFilter(
        branch="master",
        project_re="(wallet|deployment)"
    ),
    treeStableTimer=None,
    builderNames=WALLETCHANGE_TRIGGERS
))

SCHEDULERS.append(schedulers.SingleBranchScheduler(
    name="all-scheduler",
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)",
        project_re="(backoffice|wallet-core|bank|exchange|"
        "merchant|deployment|twister|sync|"
        "help|taler-merchant-demos)"
    ),
    treeStableTimer=None,
    builderNames=CODECHANGE_TRIGGERS
))

# Scheduler for all nightly builds.
SCHEDULERS.append(schedulers.Nightly(
    name="nightly-scheduler",
    builderNames=list(NIGHTLY_TRIGGERS),
    branch="master",
    hour=6,
    minute=0
))

# Provide "force" button in the web UI.
SCHEDULERS.append(schedulers.ForceScheduler(
    name="force-scheduler",
    builderNames=list(BUILDER_LIST)
))


#########################################################
####### Actual configuation initialization ##############
#########################################################

# This is the dictionary that the buildmaster pays attention to.  We also use
# a shorter alias to save typing.
c = BuildmasterConfig = {}
c["workers"] = WORKERS
c["builders"] = BUILDERS
c["schedulers"] = SCHEDULERS
c["services"] = SERVICES

# Silence warning and allow very basic phoning home.
c["buildbotNetUsageData"] = "basic"

c["title"] = "GNU Taler"
c["titleURL"] = "https://taler.net"

# This specifies what database buildbot uses to store its
# state.  You can leave  this at its default for all but the
# largest installations.
c["db"] = {
    "db_url": "sqlite:///state.sqlite",
}

# the 'change_source' setting tells the buildmaster how it should
# find out about source code changes.
c["change_source"] = [changes.PBChangeSource(user="allcs", passwd="allcs")]

# 'protocols' contains information about protocols which master
# will use for communicating with workers. You must define at
# least 'port' option that workers could connect to your master
# with this protocol. 'port' must match the value configured into
# the workers (with their --master option)
c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}}

# We use nginx to expose the BB under this URL.
c["buildbotURL"] = "https://buildbot.taler.net/"


authz = util.Authz(
    allowRules=[
        util.ForceBuildEndpointMatcher(role="admins",
                                       builder=b)
        for b in BUILDER_LIST
    ] + [
        util.StopBuildEndpointMatcher(role="admins",
                                      builder=b)
        for b in BUILDER_LIST
    ] + [
        util.RebuildBuildEndpointMatcher(role="admins",
                                         builder=b)
        for b in BUILDER_LIST
    ] + [
        util.ForceBuildEndpointMatcher(role="norole",
                                       builder=b)
        for b in BUILDER_LIST
    ] + [
        util.StopBuildEndpointMatcher(role="norole",
                                      builder=b)
        for b in BUILDER_LIST
    ] + [
        util.RebuildBuildEndpointMatcher(role="norole",
                                         builder=b)
        for b in BUILDER_LIST
    ],
    roleMatchers=[
        util.RolesFromUsername(
            roles=["admins"],
            usernames=["marcello", "florian", "christian"]
        )
    ]
)

# minimalistic config to activate new web UI
# -- formerly commented out as not packaged properly in Debian and others, see
# https://bugzilla.redhat.com/show_bug.cgi?id=1557687
c["www"] = {
    "port": 8010,
    "default_page": 'waterfall',
    "plugins": {
        "waterfall_view": True,
        "console_view": True,
        "grid_view": True,
    },
    "allowed_origins": ["https://*.taler.net"],
    "avatar_methods": [],
    "auth": util.UserPasswordAuth({"marcello":"stanisci",
                                   "florian": "dold",
                                   "christian":"grothoff"}),
    "authz": authz
}