path: root/buildbot/master.cfg

import re
from getpass import getuser
from buildbot.steps.source.git import Git
from buildbot.steps.shell import ShellCommand
from buildbot.plugins import (reporters, worker, changes, schedulers, util)

# This is a sample buildmaster config file. It must be
# installed as 'master.cfg' in your buildmaster's base
# directory.  This is the dictionary that the buildmaster
# pays attention to.  We also use a shorter alias to save
# typing.
c = BuildmasterConfig = {}

# Silence warning and allow very basic phoning home.
c["buildbotNetUsageData"] = "basic"

####### WORKERS

# The 'workers' list defines the set of recognized workers.
# Each element is a Worker object, specifying a unique worker
# name and password.  The same worker name and password must
# be configured on the worker.
c["workers"] = [

    ##
    # This worker makes the code coverage and publishes it
    # under the "lcov" Website.
    worker.Worker("lcov-worker", "lcov-pass"),

    ##
    # This worker builds manuals / API docs / tutorials.
    #
    worker.Worker("doc-worker", "doc-pass"),

    ##
    # This worker builds Websites: www and stage.
    #
    worker.Worker("sites-worker", "sites-pass"),

    ##
    # This worker builds Taler for the 'test' deployment.
    worker.Worker("test-worker", "test-pass"),

    ##
    # Tip reserve toppers.
    worker.Worker("test-topper-worker", "test-topper-pass"),
    worker.Worker("demo-topper-worker", "demo-topper-pass"),

    ##
    # This worker compiles the auditor reports for the "green"
    # demo deployment.
    worker.Worker("test-auditor-worker", "test-auditor-pass"),
    worker.Worker("demo-auditor-worker", "test-auditor-pass"),

    ##
    # This worker checks that all the services run under the
    # 'demo' deployment are up&running.
    worker.Worker("demo-worker", "demo-pass"),

    ##
    # This worker builds wallet-core.
    worker.Worker("wallet-worker", "wallet-pass"),
]

# 'protocols' contains information about protocols which master
# will use for communicating with workers. You must define at
# least 'port' option that workers could connect to your master
# with this protocol. 'port' must match the value configured into
# the workers (with their --master option)
c["protocols"] = {"pb": {"port": "tcp:9989:interface=127.0.0.1"}}

####### CHANGESOURCES

# the 'change_source' setting tells the buildmaster how it should
# find out about source code changes.

# NOTE: BB is bound to localhost
ALLCS = changes.PBChangeSource(user="allcs", passwd="allcs")

c["change_source"] = [ALLCS]

####### SCHEDULERS

# Configure the Schedulers, which decide how to react to incoming
# changes.


def twister_web_page(change):
    _change = change.asDict()
    repo = _change.get("project")
    if repo in ["www"]:
        return True

    files = _change.get("files")
    for file in files:
        if re.search(r"web", file.get("name", "")) \
                and "twister" == repo:
            return True

    return False


NIGHTLY_SCHEDULER = schedulers.Nightly(
    name="nightly-scheduler",
    builderNames=["lcov-builder",
                  "auditor-builder-demo",
                  "auditor-builder-test"],
    branch="master",
    hour=6,
    minute=0
)

DOC_SCHEDULER = schedulers.SingleBranchScheduler(
    name="periodic-doc-scheduler",
    builderNames=["doc-builder"],
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)", project_re="(docs)"
    ),
    treeStableTimer=None
)

SITES_SCHEDULER = schedulers.SingleBranchScheduler(
    name="sites-scheduler",
    builderNames=["sites-builder"],
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)",
        # Given that filter_fn is used, the line below could be
        # removed (?)
        project_re="(www|twister)",
        filter_fn=twister_web_page
    ),
    treeStableTimer=None
)

WALLET_SCHEDULER = schedulers.SingleBranchScheduler(
    name="wallet-scheduler",
    change_filter=util.ChangeFilter(
        branch="master",
        project_re="(wallet|deployment)"
    ),
    treeStableTimer=None,
    builderNames=["wallet-builder"]
)

DEMO_SERVICES_CHECKER_SCHEDULER = schedulers.Periodic(
    name="demo-services-checker-scheduler",
    periodicBuildTimer=60 * 60,  # 1 hour
    builderNames=["demo-services-checker-builder"]
)

TIP_RESERVE_TOPPER_SCHEDULER = schedulers.Periodic(
    name="tip-reserve-topper-scheduler",
    periodicBuildTimer=60 * 60 * 24 * 10,  # 10 days
    builderNames=["tip-reserve-topper-builder-demo",
                  "tip-reserve-topper-builder-test"]
)

ALL_SCHEDULER = schedulers.SingleBranchScheduler(
    name="all-scheduler",
    change_filter=util.ChangeFilter(
        branch_re="(master|stable)",
        project_re="(backoffice|wallet|bank|exchange|"
        "merchant|deployment|donations|twister|"
        "blog|help|survey|landing)"
    ),
    treeStableTimer=None,
    builderNames=["test-builder"]
)

# Scheduler monitoring the help.git repo; a forgotten repo we
# use to test BB.
DEBUG_SCHEDULER = schedulers.SingleBranchScheduler(
    name="debug-scheduler",
    change_filter=util.ChangeFilter(branch="master", project="help"),
    treeStableTimer=None,
    builderNames=["debug-builder"]
)

# Consider adding other Python parts, like the various frontends.
LINT_SCHEDULER = schedulers.SingleBranchScheduler(
    name="lint-scheduler",
    change_filter=util.ChangeFilter(
        branch="master", project_re="(bank|donations|survey|blog)"
    ),
    treeStableTimer=None,
    builderNames=["lint-builder"]
)

# Provide "force" button in the web UI.  To be removed in the
# future ?
FORCE_SCHEDULER = schedulers.ForceScheduler(
    name="force-scheduler",
    builderNames=[
        "lcov-builder",
        "auditor-builder-test",
        "auditor-builder-demo",
        "test-builder",
        "doc-builder",
        "sites-builder",
        "wallet-builder",
        "tip-reserve-topper-builder-test",
        "tip-reserve-topper-builder-demo"
    ]
)

c["schedulers"] = [
    NIGHTLY_SCHEDULER,
    TIP_RESERVE_TOPPER_SCHEDULER,
    DEMO_SERVICES_CHECKER_SCHEDULER,
    DOC_SCHEDULER,
    SITES_SCHEDULER,
    ALL_SCHEDULER,
    FORCE_SCHEDULER,
    WALLET_SCHEDULER,

    ##
    # Rarely/never used, excluding.
    # DEBUG_SCHEDULER,
    # LINT_SCHEDULER
]

####### BUILDERS

# The 'builders' list defines the Builders, which tell Buildbot
# how to perform a build: what steps, and which workers can execute
# them.  Note that any particular build will only take place on
# one worker.


def git_step(repo):
    return Git(
        repourl=repo,
        mode="full",
        method="fresh",
        logEnviron=False,
        alwaysUseLatest=True,
        haltOnFailure=True,
        branch="master"
    )


WALLET_FACTORY = util.BuildFactory()
WALLET_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git"))
WALLET_FACTORY.addStep(
    ShellCommand(
        name="fetch",
        description="Running yarn install",
        descriptionDone="Correctly installed",
        command=["npm", "install", "-g", "--prefix", "$HOME", "taler-wallet"],
        workdir="build/"
    )
)
#WALLET_FACTORY.addStep(
#    ShellCommand(
#        name="configuration",
#        description="Running configure script",
#        descriptionDone="Correctly configured",
#        command=["./configure"],
#        workdir="build/"
#    )
#)
WALLET_FACTORY.addStep(
    ShellCommand(
        name="test",
        description="Running wallet tests",
        descriptionDone="Test correctly run",
        command=["./$HOME/bin/taler-wallet-cli", "integrationtest", "--verbose"],
        workdir="build/",
        timeout=3600
    )
)
# WALLET_FACTORY.addStep(
#     ShellCommand(
#         name="lint",
#         description="Linting the wallet",
#         descriptionDone="Linting done",
#         command=["make", "lint"],
#         workdir="build/"
#     )
# )

DEBUG_FACTORY = util.BuildFactory()
DEBUG_FACTORY.addStep(
    ShellCommand(
        name="echo debug",
        description="just echoing a word",
        descriptionDone="builder responded",
        command=["echo", "I'm here!"]
    )
)


def lint_dispatcher(project):
    return "./lint_%s.sh" % project


LINT_FACTORY = util.BuildFactory()
LINT_FACTORY.addStep(
    ShellCommand(
        name="Python linter",
        description="linting Python",
        descriptionDone="linting done",
        command=util.Transform(lint_dispatcher, util.Property("project")),
        workdir="../../deployment/taler-build"
    )
)

LCOV_FACTORY = util.BuildFactory()
LCOV_FACTORY.addStep(
    ShellCommand(
        haltOnFailure=True,
        name="invalidation",
        description="Invalidating timestamps",
        descriptionDone="timestamps invalidated",
        command=["./invalidate.sh"],
        workdir="../../deployment/taler-build",
        env={"TALER_ENV_NAME": "not-test"}
    )
)  # work-around 'set -eu'
LCOV_FACTORY.addStep(
    ShellCommand(
        name="build",
        description="Compiling..",
        descriptionDone="lcov files generated",
        command=["make", "lcov"],
        workdir="../../deployment/taler-build",
        env={
            "PATH": "${HOME}/local/bin:${PATH}",
            "TALER_CHECKDB": "postgres:///talercheck-${USER}"
        }
    )
)

# FIXME: 'demo' reports generator missing.
AUDITOR_FACTORY = util.BuildFactory()
AUDITOR_FACTORY.addStep(
    ShellCommand(
        name="Auditor reports generator",
        description="Generating auditor reports.",
        descriptionDone="Auditor reports correctly generated.",
        command=["./make_auditor_reports.sh"],
        workdir="../../deployment/buildbot"
    )
)

TIP_RESERVE_TOPPER_FACTORY = util.BuildFactory()
TIP_RESERVE_TOPPER_FACTORY.addStep(
    ShellCommand(
        name="tip reserve topper",
        description="Topping the tip reserve.",
        descriptionDone="Tip reserve has been topped.",
        command=["./top_reserve.sh"],
        workdir="../../deployment/buildbot"
    )
)

BUILD_FACTORY = util.BuildFactory()
BUILD_FACTORY.addStep(
    ShellCommand(
        name="build",
        description="Building all Taler codebase.",
        descriptionDone="Taler built.",
        command=["./build.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)

BUILD_FACTORY.addStep(
    ShellCommand(
        name="config",
        description="Generating configuration file.",
        descriptionDone="Configuration file generated.",
        command=["./config.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)

BUILD_FACTORY.addStep(
    ShellCommand(
        name="keys generation and sign",
        description="Generating exchange keys, and auditor-sign them.",
        descriptionDone="Exchange keys generated, and auditor-signed.",
        command=["./keys.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'BRANCH': util.Property("branch")}
    )
)

BUILD_FACTORY.addStep(
    ShellCommand(
        name="wire details sign",
        description="Signing exchange wire details.",
        descriptionDone="Exchange wire details got signed.",
        command=["./sign.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'BRANCH': util.Property("branch")}
    )
)

BUILD_FACTORY.addStep(
    ShellCommand(
        name="restart services",
        description="Restarting inactive blue-green party.",
        descriptionDone="Restarting Taler.",
        command=["./restart.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'BRANCH': util.Property("branch")}
    )
)

BUILD_FACTORY.addStep(
    ShellCommand(
        name="check services correctly restarted",
        description="Checking services are correctly restarted.",
        descriptionDone="All services are correctly restarted.",
        command=["./checks.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        env={'DEPLOYMENT': "test"}
    )
)

SELENIUM_FACTORY = util.BuildFactory()
SELENIUM_FACTORY.addStep(
    ShellCommand(
        name="selenium",
        description="Headless browser test",
        descriptionDone="Test finished",
        command=["launch_selenium_test"],
        env={'PATH': "${HOME}/local/bin:/usr/lib/chromium:${PATH}"}
    )
)

DOC_FACTORY = util.BuildFactory()
DOC_FACTORY.addStep(
    ShellCommand(
        name="build docs",
        description="Building documentation",
        descriptionDone="Documentation built.",
        command=["./build-docs.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)

SITES_FACTORY = util.BuildFactory()
SITES_FACTORY.addStep(
    ShellCommand(
        name="build Web sites",
        description="Building all the Taler homepages",
        descriptionDone="Sites built.",
        command=["./build-sites.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True
    )
)

DEMO_SERVICES_CHECKER_FACTORY = util.BuildFactory()
DEMO_SERVICES_CHECKER_FACTORY.addStep(
    ShellCommand(
        name="demo services checker",
        description="Checking demo services are online",
        descriptionDone="Demo services are online!.",
        command=["./checks.sh"],
        workdir="../../deployment/buildbot",
        haltOnFailure=True,
        # Needed to test the 'demo' deployment.
        env={"DEPLOYMENT": "demo"}
    )
)

DEBUG_BUILDER = util.BuilderConfig(
    name="debug-builder",
    workernames=["debug-worker"],
    factory=DEBUG_FACTORY
)

LINT_BUILDER = util.BuilderConfig(
    name="lint-builder",
    workernames=["lint-worker"],
    factory=LINT_FACTORY
)

LCOV_BUILDER = util.BuilderConfig(
    name="lcov-builder",
    workernames=["lcov-worker"],
    factory=LCOV_FACTORY
)

AUDITOR_BUILDER_TEST = util.BuilderConfig(
    name="auditor-builder-test",
    workernames=["test-auditor-worker"],
    factory=AUDITOR_FACTORY
)

AUDITOR_BUILDER_DEMO = util.BuilderConfig(
    name="auditor-builder-demo",
    workernames=["demo-auditor-worker"],
    factory=AUDITOR_FACTORY
)

TIP_RESERVE_TOPPER_BUILDER_DEMO = util.BuilderConfig(
    name="tip-reserve-topper-builder-demo",
    workernames=["demo-topper-worker"],
    factory=TIP_RESERVE_TOPPER_FACTORY
)

TIP_RESERVE_TOPPER_BUILDER_TEST = util.BuilderConfig(
    name="tip-reserve-topper-builder-test",
    workernames=["test-topper-worker"],
    factory=TIP_RESERVE_TOPPER_FACTORY
)

DEMO_SERVICES_CHECKER_BUILDER = util.BuilderConfig(
    name="demo-services-checker-builder",
    workernames="demo-worker",
    factory=DEMO_SERVICES_CHECKER_FACTORY
)

TEST_BUILDER = util.BuilderConfig(
    name="test-builder", workernames=["test-worker"], factory=BUILD_FACTORY
)

SELENIUM_BUILDER = util.BuilderConfig(
    name="selenium-builder",
    workernames=["selenium-worker"],
    factory=SELENIUM_FACTORY
)

DOC_BUILDER = util.BuilderConfig(
    name="doc-builder", workernames=["doc-worker"], factory=DOC_FACTORY
)

SITES_BUILDER = util.BuilderConfig(
    name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY
)

WALLET_BUILDER = util.BuilderConfig(
    name="wallet-builder",
    workernames=["wallet-worker"],
    factory=WALLET_FACTORY
)

c["builders"] = [
    LCOV_BUILDER,
    AUDITOR_BUILDER_TEST,
    AUDITOR_BUILDER_DEMO,
    TEST_BUILDER,
    TIP_RESERVE_TOPPER_BUILDER_DEMO,
    TIP_RESERVE_TOPPER_BUILDER_TEST,
    DEMO_SERVICES_CHECKER_BUILDER,
    DOC_BUILDER,
    SITES_BUILDER,
    WALLET_BUILDER,
]

####### BUILDBOT SERVICES

# 'services' is a list of BuildbotService items like reporter
# targets. The status of each build will be pushed to these
# targets. buildbot/reporters/*.py has a variety to choose from,
# like IRC bots.

IRC = reporters.IRC(
    "irc.eu.freenode.net",
    "taler-bb",
    useColors=False,
    channels=[{
        "channel": "#taler"
    }],
    password="taler-bb-pass19",
    notify_events={
        'exception': 1,
        'successToFailure': 1,
        'failureToSuccess': 1
    }
)

EMAIL = reporters.MailNotifier(
    fromaddr="testbuild@taler.net",
    sendToInterestedUsers=False,
    # notify from pass to fail, and viceversa.
    mode=("change"),
    builders=(
        "lcov-builder",
        "doc-builder",
        "test-builder",
        ##
        # Rarely/never used, excluding.
        # "wallet-builder",
        # "selenium-builder"
    ),
    extraRecipients=["buildfailures@taler.net"],
    subject="Taler build."
)

c["services"] = [IRC, EMAIL]

####### PROJECT IDENTITY

c["title"] = "Taler"
c["titleURL"] = "https://taler.net"

# We use nginx to expose the BB under this URL.
c["buildbotURL"] = "https://buildbot.taler.net/"

from taler_bb_userpass_db import USER_PASSWORD_DB

BUILDER_LIST = [
    "doc-builder",
    "lcov-builder",
    "auditor-builder",
    "auditor-builder-test",
    "auditor-builder-demo",
    "demo-services-checker-builder",
    "tip-reserve-topper-builder-demo",
    "tip-reserve-topper-builder-test",
    "sites-builder",
    "test-builder",
    "wallet-builder",
]

authz = util.Authz(
    allowRules=[
        util.ForceBuildEndpointMatcher(role="admins",
                                       builder=b)
        for b in BUILDER_LIST
    ] + [
        util.StopBuildEndpointMatcher(role="admins",
                                      builder=b)
        for b in BUILDER_LIST
    ] + [
        util.RebuildBuildEndpointMatcher(role="admins",
                                         builder=b)
        for b in BUILDER_LIST
    ] + [
        util.ForceBuildEndpointMatcher(role="norole",
                                       builder=b)
        for b in BUILDER_LIST
    ] + [
        util.StopBuildEndpointMatcher(role="norole",
                                      builder=b)
        for b in BUILDER_LIST
    ] + [
        util.RebuildBuildEndpointMatcher(role="norole",
                                         builder=b)
        for b in BUILDER_LIST
    ],
    roleMatchers=[
        util.RolesFromUsername(
            roles=["admins"],
            usernames=["marcello", "florian", "christian"]
        )
    ]
)

# minimalistic config to activate new web UI
c["www"] = {
    "port": 8010,
    "plugins": {
        "waterfall_view": {},
        "console_view": {}
    },
    "allowed_origins": ["https://*.taler.net"],
    "avatar_methods": [],
    "auth": util.UserPasswordAuth(USER_PASSWORD_DB),
    "authz": authz
}

####### DB URL

c["db"] = {
    # This specifies what database buildbot uses to store its
    # state.  You can leave  this at its default for all but the
    # largest installations.
    "db_url": "sqlite:///state.sqlite",
}