diff options
author | Devan Carpenter <devan@taler.net> | 2023-06-12 14:53:07 -0400 |
---|---|---|
committer | Devan Carpenter <devan@taler.net> | 2023-06-12 21:49:25 -0400 |
commit | 2f2372fd65f5bba7f9ab2dcfbe0916016dc2ccec (patch) | |
tree | e38eaa284edc724541a7dc831f0d9a06b972508a | |
parent | 8d0574d7313882e8a75d0e21ee9bafc048230fb1 (diff) | |
download | deployment-2f2372fd65f5bba7f9ab2dcfbe0916016dc2ccec.tar.gz deployment-2f2372fd65f5bba7f9ab2dcfbe0916016dc2ccec.tar.bz2 deployment-2f2372fd65f5bba7f9ab2dcfbe0916016dc2ccec.zip |
buildbot: lint and cleanup imports
flake8 linting, and make imports explicit. this allows for better static
analysis and linting.
-rw-r--r-- | buildbot/master.cfg | 116 |
1 files changed, 64 insertions, 52 deletions
diff --git a/buildbot/master.cfg b/buildbot/master.cfg index 734831e..b2f980a 100644 --- a/buildbot/master.cfg +++ b/buildbot/master.cfg @@ -26,12 +26,14 @@ # @author Devan Carpenter import re -from getpass import getuser - from buildbot.steps.source.git import Git from buildbot.steps.shell import ShellCommand -from buildbot.plugins import * +from buildbot.plugins import reporters +from buildbot.plugins import schedulers +from buildbot.plugins import steps +from buildbot.plugins import util from buildbot.reporters.generators.build import BuildStatusGenerator +from buildbot.worker import Worker # This is a sample buildmaster config file. It must be # installed as 'master.cfg' in your buildmaster's base @@ -88,6 +90,7 @@ class MessageFormatterWithStdout(reporters.MessageFormatter): # subject="tips availability on demo") # ) + SERVICES = [] # The 'builders' list defines the Builders, which tell Buildbot @@ -104,13 +107,15 @@ SCHEDULERS = [] NIGHTLY_TRIGGERS=[] # Array of builders to be scheduled whenever any of the code Git repos change -CODECHANGE_TRIGGERS=[] +CODECHANGE_TRIGGERS = [] -# Array of builders to be scheduled whenever the wallet-core or deployment change -WALLETCHANGE_TRIGGERS=[] +# Array of builders to be scheduled whenever the wallet-core or +# deployment change +WALLETCHANGE_TRIGGERS = [] -# Array of builder names for which build status reports should be sent via e-mail -EMAIL_ALERTS=[] +# Array of builder names for which build status reports should be sent +# via e-mail +EMAIL_ALERTS = [] ############ Convenience functions ################# @@ -118,13 +123,14 @@ EMAIL_ALERTS=[] # Create a FACTORY with a deployment.git checkout as the first step. def create_factory_with_deployment(): f = util.BuildFactory() - update_deployment (f) + update_deployment(f) return f + # Convenience function that checks out a Git repository. # First argument is the URL of the Git to clone, second # the desired branch. Default is 'master'. -def git_step(repo,target_branch="master"): +def git_step(repo, target_branch="master"): return Git( repourl=repo, mode="full", @@ -135,9 +141,10 @@ def git_step(repo,target_branch="master"): branch=target_branch ) + # Convenience function that runs 'make check' in a # directory of the code inside of a netjail. -def jailed_check(package,srcdirs): +def jailed_check(package, srcdirs): return steps.ShellSequence( name="Tests of " + package, description="Testing " + package, @@ -147,6 +154,7 @@ def jailed_check(package,srcdirs): workdir="../../sources/" + package ) + # Convenience function that checks out the deployment. def update_deployment(factory): factory.addStep(steps.ShellSequence( @@ -172,8 +180,9 @@ def container_run_step(stepName, factory, WORK_DIR, containerName, util.ShellArg(command=["podman", "build", "-t", containerName, "-f", containerFile, "."], logname='build container', haltOnFailure=True), - util.ShellArg(command=["podman", "run", "-ti", "--rm", "--volume", - f"{WORK_DIR}:/workdir", "--workdir", "/workdir", + util.ShellArg(command=["podman", "run", "-ti", "--rm", + "--volume", f"{WORK_DIR}:/workdir", + "--workdir", "/workdir", containerName, jobCmd], logname='run inside container', haltOnFailure=True), ], @@ -196,11 +205,12 @@ def container_run_step(stepName, factory, WORK_DIR, containerName, ################ 1: BUILDMASTER JOB ################################### + ## # This worker restarts the buildmaster itself on # changes to this file. # Location: /home/buildbot-master @ taler.net -WORKERS.append(worker.Worker("buildmaster-worker", "buildmaster-pass")) +WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) BUILDMASTER_FACTORY = create_factory_with_deployment() BUILDMASTER_FACTORY.addStep( @@ -238,7 +248,7 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( ## # This worker builds manuals / API docs / tutorials. # Location: /home/docbuilder @ taler.net -WORKERS.append(worker.Worker("doc-worker", "doc-pass")) +WORKERS.append(Worker("doc-worker", "doc-pass")) DOC_FACTORY = create_factory_with_deployment() @@ -344,21 +354,21 @@ DOC_FACTORY.addStep( ) ) ## Disabled wallet doxygen job temporarily, until buildbot config refactored -#DOC_FACTORY.addStep( -# steps.ShellSequence( -# name="doxygen::wallet", -# description="building wallet typescript documentation", -# descriptionDone="typedoc on taler-wallet-core finished", -# commands=[ -# util.ShellArg(command=["./bootstrap"], logname="bootstrap"), -# util.ShellArg(command=["./configure"], logname="configure"), -# util.ShellArg(command=["make"], logname="make"), -# util.ShellArg(command=["pnpm", "install", "-W", "typedoc"], logname="pnpm"), -# util.ShellArg(command=["./node_modules/typedoc/bin/typedoc", "--out", "dist/typedoc", "--tsconfig", "tsconfig.build.json", "packages/taler-util/src/index.ts", "packages/taler-wallet-cli/src/index.ts", "packages/taler-wallet-android/src/index.ts", "packages/taler-wallet-core/src/index.ts" ], logname="typedoc"), -# ], -# workdir="../../sources/wallet-core/" -# ) -#) +# DOC_FACTORY.addStep( +# steps.ShellSequence( +# name="doxygen::wallet", +# description="building wallet typescript documentation", +# descriptionDone="typedoc on taler-wallet-core finished", +# commands=[ +# util.ShellArg(command=["./bootstrap"], logname="bootstrap"), +# util.ShellArg(command=["./configure"], logname="configure"), +# util.ShellArg(command=["make"], logname="make"), +# util.ShellArg(command=["pnpm", "install", "-W", "typedoc"], logname="pnpm"), +# util.ShellArg(command=["./node_modules/typedoc/bin/typedoc", "--out", "dist/typedoc", "--tsconfig", "tsconfig.build.json", "packages/taler-util/src/index.ts", "packages/taler-wallet-cli/src/index.ts", "packages/taler-wallet-android/src/index.ts", "packages/taler-wallet-core/src/index.ts" ], logname="typedoc"), +# ], +# workdir="../../sources/wallet-core/" +# ) +# ) DOC_FACTORY.addStep( steps.ShellSequence( name="prepare challenger", @@ -421,7 +431,7 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( ## # This worker builds Websites: www and stage. # -WORKERS.append(worker.Worker("sites-worker", "sites-pass")) +WORKERS.append(Worker("sites-worker", "sites-pass")) SITES_FACTORY = create_factory_with_deployment() SITES_FACTORY.addStep( @@ -441,6 +451,7 @@ BUILDERS.append(util.BuilderConfig( EMAIL_ALERTS.append("sites-builder") + # The web page changed if 'www' changed OR if 'web' in the 'twister' repo changed: def twister_web_page(change): _change = change.asDict() @@ -455,6 +466,7 @@ def twister_web_page(change): return True return False + # Sites are re-build whenever deployment, www buywith, or twister changes. SCHEDULERS.append(schedulers.SingleBranchScheduler( name="sites-scheduler", @@ -472,7 +484,7 @@ SCHEDULERS.append(schedulers.SingleBranchScheduler( ## # This worker makes the code coverage and publishes it # under the "lcov" Website. -WORKERS.append(worker.Worker("lcov-worker", "lcov-pass")) +WORKERS.append(Worker("lcov-worker", "lcov-pass")) LCOV_FACTORY = create_factory_with_deployment() LCOV_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) @@ -523,7 +535,7 @@ NIGHTLY_TRIGGERS.append("taler-demo-healthcheck-builder") ## # This worker builds everything and runs our 'make check' # test suite against 'everything'. -WORKERS.append(worker.Worker("checker-worker", "checker-pass")) +WORKERS.append(Worker("checker-worker", "checker-pass")) INTEGRATIONTEST_FACTORY = create_factory_with_deployment() INTEGRATIONTEST_FACTORY.addStep( @@ -581,21 +593,21 @@ INTEGRATIONTEST_FACTORY.addStep( haltOnFailure=False ) ) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("libmicrohttpd", [ +INTEGRATIONTEST_FACTORY.addStep(jailed_check("libmicrohttpd", [ "microhttpd", "testcurl", "testzzuf", ])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("gnunet", [ +INTEGRATIONTEST_FACTORY.addStep(jailed_check("gnunet", [ "util", "pq", "curl", "json", ])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("twister", [ +INTEGRATIONTEST_FACTORY.addStep(jailed_check("twister", [ "test", ])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", [ +INTEGRATIONTEST_FACTORY.addStep(jailed_check("exchange", [ "util", "curl", "mhd", @@ -611,7 +623,7 @@ INTEGRATIONTEST_FACTORY.addStep(jailed_check ("exchange", [ "testing", "benchmark", ])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", [ +INTEGRATIONTEST_FACTORY.addStep(jailed_check("merchant", [ "mustach", "backenddb", "backend", @@ -619,7 +631,7 @@ INTEGRATIONTEST_FACTORY.addStep(jailed_check ("merchant", [ "testing", "merchant-tools", ])) -INTEGRATIONTEST_FACTORY.addStep(jailed_check ("sync", [ +INTEGRATIONTEST_FACTORY.addStep(jailed_check("sync", [ "util", "syncdb", "sync", @@ -682,7 +694,7 @@ CODECHANGE_TRIGGERS.append("checker-builder") ## # linkchecker worker checks for dead links in the Website # Location: /home/linkchecker @ taler.net -WORKERS.append(worker.Worker("linkchecker-worker", "linkchecker-pass")) +WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) # linkchecker FACTORY LINKCHECKER_FACTORY = create_factory_with_deployment() @@ -706,7 +718,7 @@ BUILDERS.append(util.BuilderConfig( )) docs_generator = BuildStatusGenerator( - mode=('change','problem','failing','exception',), + mode=('change', 'problem', 'failing', 'exception',), builders=[ 'linkchecker-builder', ], @@ -715,7 +727,7 @@ docs_generator = BuildStatusGenerator( wantSteps=True, wantLogs=True ), - add_logs=True + add_logs=True ) @@ -727,7 +739,7 @@ SERVICES.append(reporters.MailNotifier( extraRecipients=['linkcheck@taler.net'] )) -#SERVICES.append(tipReserveEmails) +# SERVICES.append(tipReserveEmails) NIGHTLY_TRIGGERS.append("linkchecker-builder") @@ -737,7 +749,7 @@ NIGHTLY_TRIGGERS.append("linkchecker-builder") ## # codespell worker checks for spelling mistakes in code # Location: /home/codespell @ taler.net -WORKERS.append(worker.Worker("codespell-worker", "codespell-pass")) +WORKERS.append(Worker("codespell-worker", "codespell-pass")) CODESPELL_FACTORY = create_factory_with_deployment() CODESPELL_FACTORY.addStep( @@ -843,7 +855,7 @@ CODECHANGE_TRIGGERS.append("codespell-builder") -# WORKERS.append(worker.Worker("demo-worker", "demo-pass")) +# WORKERS.append(Worker("demo-worker", "demo-pass")) # DEMO_SERVICES_INTEGRATIONTEST_FACTORY = create_factory_with_deployment() # DEMO_SERVICES_INTEGRATIONTEST_FACTORY.addStep( # ShellCommand( @@ -882,7 +894,7 @@ CODECHANGE_TRIGGERS.append("codespell-builder") ## # health checks performed by wallet-cli for demo -WORKERS.append(worker.Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass")) +WORKERS.append(Worker("taler-demo-healthcheck", "taler-demo-healthcheck-pass")) TALER_DEMO_HEALTHCHECK_FACTORY = create_factory_with_deployment() # For the moment, the health-check uses whatever wallet @@ -908,7 +920,7 @@ BUILDERS.append(util.BuilderConfig( ## # health checks performed by wallet-cli for test -WORKERS.append(worker.Worker("taler-test-healthcheck", "taler-test-healthcheck-pass")) +WORKERS.append(Worker("taler-test-healthcheck", "taler-test-healthcheck-pass")) TALER_TEST_HEALTHCHECK_FACTORY = create_factory_with_deployment() TALER_TEST_HEALTHCHECK_FACTORY.addStep(git_step("git://git.taler.net/wallet-core.git")) @@ -954,7 +966,7 @@ WALLETCHANGE_TRIGGERS.append("taler-test-healthcheck-builder") ## # testing buildbot using the "buildslavetest" user (for no specific reason except it exists) # Location: /home/buidlslavetest @ taler.net -WORKERS.append(worker.Worker("buildslavetest-worker", "Gei8naiyox4uuhoo")) +WORKERS.append(Worker("buildslavetest-worker", "Gei8naiyox4uuhoo")) BUILD_FACTORY = create_factory_with_deployment() BUILD_FACTORY.addStep( @@ -1074,7 +1086,7 @@ SELENIUM_FACTORY.addStep( # URL: https://git.taler.net/deployment.git/tree/docker/compile-and-check # 01/18/2022 - Added by Javier Sepulveda -WORKERS.append(worker.Worker("compilecheck-worker", "compilecheck-pass")) +WORKERS.append(Worker("compilecheck-worker", "compilecheck-pass")) # check-and-compile FACTORY - Step 1 - Build docker image @@ -1124,7 +1136,7 @@ EMAIL_ALERTS.append("compilecheck-builder") # URL: https://git.taler.net/deployment.git/tree/packaging # 01/23/2022 - Added by Javier Sepulveda -WORKERS.append(worker.Worker("packaging-worker", "packaging-pass")) +WORKERS.append(Worker("packaging-worker", "packaging-pass")) # check-and-compile FACTORY - Step 1 - Build docker image @@ -1248,8 +1260,8 @@ for reponame in container_repos: builderNames=[f"{reponame}-builder"] )) -## TODO: TBD -#EMAIL_ALERTS.append("container-builder") +# TODO: TBD +# EMAIL_ALERTS.append("container-builder") |