master.cfg (52554B)
1 # -*- python -*- 2 # ex: set syntax=python: 3 4 ## 5 # This file is part of TALER 6 # (C) 2016-2025 Taler Systems SA 7 # 8 # TALER is free software; you can redistribute it and/or 9 # modify it under the terms of the GNU Affero General Public 10 # License as published by the Free Software Foundation; either 11 # version 3, or (at your option) any later version. 12 # 13 # TALER is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty 15 # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 16 # See the GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public 19 # License along with TALER; see the file COPYING. If not, 20 # see <http://www.gnu.org/licenses/> 21 # 22 # @author Florian Dold 23 # @author Marcello Stanisci 24 # @author ng0 25 # @author Christian Grothoff 26 # @author Devan Carpenter 27 import ast 28 import configparser 29 import glob 30 import os 31 import pathlib 32 import pwd 33 import re 34 import subprocess 35 36 from buildbot.changes.pb import PBChangeSource 37 from buildbot.steps.source.git import Git 38 from buildbot.steps.shell import ShellCommand 39 from buildbot.plugins import changes 40 from buildbot.plugins import reporters 41 from buildbot.plugins import schedulers 42 from buildbot.plugins import steps 43 from buildbot.plugins import secrets, util 44 from buildbot.process import buildstep, logobserver 45 from buildbot.process.results import SKIPPED 46 from buildbot.reporters.generators.build import BuildStatusGenerator 47 from buildbot.worker import Worker 48 from twisted.internet import defer 49 50 # This is a sample buildmaster config file. It must be 51 # installed as 'master.cfg' in your buildmaster's base 52 # directory. 53 54 # This file has the following structure: 55 # - Globals: definition of global variables we use throughout 56 # + Convenience functions: helper functions useful for many jobs 57 # - Jobs: actual job definitions 58 # - General purpose: triggers and alerts shared by various jobs 59 # + general purpose notification (alerts) 60 # + general purpose triggers (schedulers) 61 # - Actual buildbot configuration object initialization 62 63 ################################################################# 64 ######################### GLOBALS ############################### 65 ################################################################# 66 67 # The 'workers' list defines the set of recognized workers. 68 # Each element is a Worker object, specifying a unique worker 69 # name and password. The same worker name and password must 70 # be configured on the worker. 71 WORKERS = [] 72 73 # 'services' is a list of BuildbotService items like reporter 74 # targets. The status of each build will be pushed to these 75 # targets. buildbot/reporters/*.py has a variety to choose from, 76 # like IRC bots. 77 78 79 class MessageFormatterWithStdout(reporters.MessageFormatter): 80 def buildAdditionalContext(self, master, ctx): 81 stdout = [] 82 for step in ctx["build"]["steps"]: 83 for log in step["logs"]: 84 all_logs = log["content"]["content"].splitlines() 85 # Including only what the script printed on stdout. 86 for line in all_logs: 87 if re.search("^o", line): 88 stdout.append(line[1:]) 89 ctx.update(dict(stdout="\n".join(stdout))) 90 91 92 ##################################################### 93 # Commit message triggers # 94 # # 95 # This checks for triggers in the commit messages # 96 # !tarball will trigger a release build # 97 # !coverity will trigger a coverity check # 98 ##################################################### 99 def checkForTarballTrigger(change): 100 if "!tarball" in change.comments: 101 return True 102 return False 103 def checkForCoverityTrigger(change): 104 if "!coverity" in change.comments: 105 return True 106 107 SERVICES = [] 108 109 # The 'builders' list defines the Builders, which tell Buildbot 110 # how to perform a build: what steps, and which workers can execute 111 # them. Note that any particular build will only take place on 112 # one worker. 113 BUILDERS = [] 114 115 # Configures the Schedulers, which decide how to react to incoming 116 # changes. 117 SCHEDULERS = [] 118 119 # Array of builders to be scheduled every night. 120 NIGHTLY_TRIGGERS=[] 121 122 # Array of builders to be scheduled whenever any of the code Git repos change 123 CODECHANGE_TRIGGERS = [] 124 125 # Array of builders to be scheduled whenever the taler-typescript-core or 126 # taler-deployment change 127 WALLETCHANGE_TRIGGERS = [] 128 129 # Array of builder names for which build status reports should be sent 130 # via e-mail 131 EMAIL_ALERTS = [] 132 133 # Array of email address for which build status reports shoudl be sent 134 BUILDER_EMAIL_ADDRESSES = [] 135 136 ############ Convenience functions ################# 137 138 # Create a FACTORY with a taler-deployment.git checkout as the first step. 139 def create_factory_with_deployment(): 140 f = util.BuildFactory() 141 update_deployment(f) 142 return f 143 144 # Convenience function that checks out a Git repository. 145 # First argument is the URL of the Git to clone, second 146 # the desired branch. Default is 'master'. 147 def git_step(repo, target_branch="master"): 148 return Git( 149 repourl=repo, 150 mode="full", 151 method="fresh", 152 logEnviron=False, 153 alwaysUseLatest=True, 154 haltOnFailure=True, 155 branch=target_branch 156 ) 157 158 # FIXME: document this function! 159 def add_default_step(bldr, step): 160 worker_script = ".buildbot/%(prop:workername)s_" + step + ".sh" 161 default_script = ".buildbot/" + step + ".sh" 162 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (worker_script,worker_script) 163 bldr.addStep(steps.SetPropertyFromCommand(command=util.Interpolate(cmd), 164 hideStepIf=True, 165 property='buildbotScript_%s' % step, 166 name="Checking for worker-specific %s script" % step)) 167 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (default_script,default_script) 168 bldr.addStep(steps.SetPropertyFromCommand(doStepIf=(util.Property('buildbotScript_%s' % step) != util.Interpolate(worker_script)), 169 hideStepIf=True, 170 command=util.Interpolate(cmd), 171 property='buildbotScript_%s' % step, 172 name="Checking for %s script" % step)) 173 bldr.addStep(steps.ShellCommand(command=util.Property('buildbotScript_%s' % step), 174 haltOnFailure=True, 175 env={'PATH': "${HOME}/.local/bin:$HOME/bin:${PATH}"}, 176 doStepIf=(util.Property('buildbotScript_%s' % step) != None), 177 hideStepIf=lambda results, s: results==SKIPPED, 178 name="Executing %s step" % step)) 179 180 181 182 # Convenience function that runs 'make check' in a 183 # directory of the code inside of a netjail. 184 def jailed_check(package, srcdirs): 185 return steps.ShellSequence( 186 name="Tests of " + package, 187 description="Testing " + package, 188 descriptionDone="Pass", 189 commands=map(lambda srcdir: util.ShellArg(command=["sudo", "/usr/local/bin/netjail.sh", "/home/container-worker/taler-deployment/buildbot/with-postgres.sh", "bash", "-c", "'cd src/"+srcdir+" make install check'"]), srcdirs), 190 env={'PATH': "${HOME}/local/bin:${PATH}"}, 191 workdir="../../sources/" + package 192 ) 193 194 # Convenience function that checks out the deployment. 195 def update_deployment(factory): 196 factory.addStep(steps.ShellSequence( 197 name="update taler-deployment", 198 description="removing old deployment and fetching fresh repository", 199 descriptionDone="Deployment updated", 200 commands=[ 201 util.ShellArg(command=["rm", "-rf", "taler-deployment"]), 202 util.ShellArg(command=["git", "clone", "git://git.taler.net/taler-deployment"]), 203 ], 204 haltOnFailure=True, 205 workdir="../.." 206 )) 207 208 # Convenience function that builds and runs a container. 209 def container_add_step(HALT_ON_FAILURE, 210 WARN_ON_FAILURE, 211 CONTAINER_BUILD, 212 CONTAINER_NAME, 213 factory, 214 WORK_DIR, 215 repoName, 216 stepName, 217 CONTAINER_ARCH="amd64", 218 jobCmd="/workdir/contrib/ci/ci.sh", 219 containerFile="contrib/ci/Containerfile"): 220 print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") 221 222 runCommand = ["podman", "--transient-store", "run", "--rm", 223 "--arch", CONTAINER_ARCH, 224 "--log-driver=none", 225 "--add-host", "taler.host.internal:10.0.2.2", 226 "--network", "slirp4netns:allow_host_loopback=true", 227 "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), 228 "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), 229 "--env", util.Interpolate("CI_PROJECT_NAME=%(src::project)s"), 230 "--cap-add", "SYS_ADMIN,CAP_SYS_CHROOT", 231 "--volume", f"{WORK_DIR}:/workdir", 232 "--volume", "/home/container-worker/ephemeral_ci_artifacts:/artifacts", 233 "--volume", "/home/container-worker/persistent_ci_artifacts:/persistent_artifacts", 234 "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", 235 "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", 236 "--workdir", "/workdir"] 237 238 # Inputs directory 239 inputs_path = f"/home/container-worker/container_inputs/{repoName}" 240 print(f"Checking that {inputs_path} exists") 241 if os.path.isdir(inputs_path): 242 print(f"Adding {inputs_path}") 243 runCommand += ["--volume", f"{inputs_path}:/inputs:ro"] 244 else: 245 print(f"Inputs directory not found at {inputs_path}") 246 247 if CONTAINER_BUILD: 248 runCommand += ["--volume", f"/run/user/{pwd.getpwnam('container-worker').pw_uid}/podman/podman.sock:/run/podman/podman.sock", 249 "--security-opt", "label=disable"] 250 251 runCommand += [CONTAINER_NAME, jobCmd] 252 253 runArg = util.ShellArg(command=runCommand, 254 logname='run inside container', 255 haltOnFailure=HALT_ON_FAILURE) 256 257 buildArg = util.ShellArg(command=["podman", "build", "-t", CONTAINER_NAME, 258 "--arch", CONTAINER_ARCH, 259 "-f", containerFile, "."], 260 logname='build container', haltOnFailure=True) 261 262 if not CONTAINER_BUILD: 263 return steps.ShellSequence( 264 name=stepName, 265 commands=[runArg], 266 haltOnFailure=HALT_ON_FAILURE, 267 workdir=WORK_DIR 268 ) 269 else: 270 return steps.ShellSequence( 271 name=stepName, 272 commands=[buildArg, runArg], 273 haltOnFailure=HALT_ON_FAILURE, 274 workdir=WORK_DIR 275 ) 276 277 ################################################################## 278 ######################## JOBS #################################### 279 ################################################################## 280 281 # For every job, we have (in this order!): 282 # - worker(s): hosts/users that run the job 283 # - factory: list of steps that define what to do 284 # - builder: gives the job a name and binds it to the factory and worker 285 # - (OPTIONAL) alerts: notifications to trigger when the job fails 286 # Pre-defined: EMAIL_ALERTS 287 # - scheduler: rules that define when to run the job 288 # Pre-defined: NIGHTLY_TRIGGERS, CODECHANGE_TRIGGERS, WALLETCHANGE_TRIGGERS 289 290 ################ 1: BUILDMASTER JOB ################################### 291 292 ## 293 # This worker restarts the buildmaster itself on 294 # changes to this file. 295 # Location: /home/buildbot-master 296 WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) 297 298 BUILDMASTER_FACTORY = create_factory_with_deployment() 299 BUILDMASTER_FACTORY.addStep( 300 ShellCommand( 301 name="restart buildmaster", 302 description="trigger buildmaster restart with new configuration", 303 descriptionDone="Buildmaster updated", 304 command=["pkill", "-HUP", "-A", "-u", "buildbot-master", "-f", "/usr/bin/python3"], 305 workdir="../.." 306 ) 307 ) 308 309 BUILDERS.append(util.BuilderConfig( 310 name="buildmaster-builder", 311 workernames=["buildmaster-worker"], 312 factory=BUILDMASTER_FACTORY 313 )) 314 315 EMAIL_ALERTS.append("buildmaster-builder") 316 317 # Buildmaster is notified whenever deployment.git changes 318 SCHEDULERS.append(schedulers.SingleBranchScheduler( 319 name="buildmaster-scheduler", 320 change_filter=util.ChangeFilter( 321 branch="master", 322 project_re="(taler-deployment)" 323 ), 324 treeStableTimer=None, 325 builderNames=["buildmaster-builder"] 326 )) 327 328 ################ 2: WEBSITE JOB ################################### 329 330 ## 331 # This worker builds Websites: www and stage. 332 # 333 WORKERS.append(Worker("sites-worker", "sites-pass")) 334 335 SITES_FACTORY = create_factory_with_deployment() 336 SITES_FACTORY.addStep( 337 ShellCommand( 338 name="build Web sites", 339 description="Building all the Taler homepages", 340 descriptionDone="Sites built.", 341 command=["./build-sites.sh"], 342 workdir="../../taler-deployment/worker-sites", 343 haltOnFailure=True 344 ) 345 ) 346 347 BUILDERS.append(util.BuilderConfig( 348 name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY 349 )) 350 351 #EMAIL_ALERTS.append("sites-builder") 352 353 354 # The web page changed if 'taler-www', 'taler-tutorials', 355 # 'taler-docs' or 'twister' changed 356 def web_page(change): 357 _change = change.asDict() 358 repo = _change.get("project") 359 if repo in ["taler-docs", "taler-tutorials", "taler-www", "twister", "taler-deployment", "buywith", "taler-ops-www", "taler-systems-www", "anastasis-www"]: 360 return True 361 return False 362 363 364 # Sites are re-build whenever taler-deployment, taler-www, or twister changes. 365 SCHEDULERS.append(schedulers.SingleBranchScheduler( 366 name="sites-scheduler", 367 builderNames=["sites-builder"], 368 change_filter=util.ChangeFilter( 369 branch_re="(master|stable)", 370 filter_fn=web_page 371 ), 372 treeStableTimer=None 373 )) 374 375 376 ################ 3: 'check links' JOB ################################### 377 378 ## 379 # linkchecker worker checks for dead links in the Website 380 # Location: /home/linkchecker-worker 381 WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) 382 383 # linkchecker FACTORY 384 LINKCHECKER_FACTORY = create_factory_with_deployment() 385 LINKCHECKER_FACTORY.addStep( 386 ShellCommand( 387 name="linkchecker", 388 description="Check taler.net website for broken links && Notify", 389 descriptionDone="Results of wget in buildbot logs.", 390 command=["/home/linkchecker-worker/taler-deployment/worker-linkchecker/linkchecker.sh"], 391 workdir="/home/linkchecker-worker", 392 haltOnFailure=True, 393 timeout=7200 # 2 hours 394 ) 395 ) 396 397 # linkchecker BUILDER 398 # worker at linkchecker@taler.net 399 BUILDERS.append(util.BuilderConfig( 400 name="linkchecker-builder", 401 workernames="linkchecker-worker", 402 factory=LINKCHECKER_FACTORY 403 )) 404 405 docs_generator = BuildStatusGenerator( 406 mode=('change', 'problem', 'failing', 'exception',), 407 builders=[ 408 'linkchecker-builder', 409 ], 410 message_formatter=reporters.MessageFormatter( 411 template_type='plain', 412 want_logs_content=True 413 ) 414 ) 415 416 417 SERVICES.append(reporters.MailNotifier( 418 fromaddr="bb@taler.net", 419 generators=[docs_generator], 420 sendToInterestedUsers=False, 421 useTls=False, 422 relayhost="localhost", 423 smtpPort=25, 424 dumpMailsToLog=True, 425 extraRecipients=['linkcheck@taler.net'] 426 )) 427 428 # SERVICES.append(tipReserveEmails) 429 430 NIGHTLY_TRIGGERS.append("linkchecker-builder") 431 432 #################### 433 ## GNUnet workers ## 434 #################### 435 436 WORKERS.append(Worker("firefly-x86_64-amdepyc", "pass")) 437 WORKERS.append(Worker("mp-amd64-openbsd", "Tai7zeic")) 438 WORKERS.append(Worker("schanzen-aarch64-fedora-meson", "eWi9keet")) 439 440 441 SCHEDULERS.append(schedulers.AnyBranchScheduler( 442 name="gnunet", 443 change_filter=util.ChangeFilter(branch_re='master', 444 repository='git://git.gnunet.org/gnunet.git'), 445 treeStableTimer=None, 446 builderNames=["gnunet-debian-x86_64", 447 "gnunet-fedora-aarch64"])) 448 449 SCHEDULERS.append(schedulers.AnyBranchScheduler( 450 name="gnunet-dev", 451 change_filter=util.ChangeFilter(branch_re='dev/.+', 452 repository='git://git.gnunet.org/gnunet.git'), 453 treeStableTimer=None, 454 builderNames=["gnunet-debian-x86_64-dev", 455 "gnunet-fedora-aarch64-dev"])) 456 457 SCHEDULERS.append(schedulers.SingleBranchScheduler( 458 name="tagged_release", 459 change_filter=util.ChangeFilter(branch_re='.*v[0-9]*[.][0-9]*[.][0-9]*$', 460 repository='git://git.gnunet.org/gnunet.git'), 461 treeStableTimer=None, 462 builderNames=["gnunet_release"])) 463 464 SCHEDULERS.append(schedulers.SingleBranchScheduler( 465 name="tarball", 466 onlyImportant=True, 467 change_filter=util.ChangeFilter(branch='master', 468 repository='git://git.gnunet.org/gnunet.git'), 469 fileIsImportant=checkForTarballTrigger, 470 treeStableTimer=None, 471 builderNames=["gnunet_release"])) 472 473 474 # Build a tarball nightly 475 SCHEDULERS.append(schedulers.Nightly(name='nightly', 476 change_filter=util.ChangeFilter(branch='master', 477 repository='git://git.gnunet.org/gnunet.git'), 478 builderNames=['gnunet_release'], 479 onlyIfChanged=True, 480 hour=6, minute=0)) 481 482 SCHEDULERS.append(schedulers.SingleBranchScheduler( 483 name="gnunet_cov", 484 onlyImportant=True, 485 change_filter=util.ChangeFilter(branch='master', 486 repository='git://git.gnunet.org/gnunet.git'), 487 fileIsImportant=checkForCoverityTrigger, 488 treeStableTimer=None, 489 builderNames=["gnunet_coverity"])) 490 491 SCHEDULERS.append(schedulers.SingleBranchScheduler( 492 name="gnunet_rpm", 493 change_filter=util.ChangeFilter(branch='dev/schanzen/copr', 494 repository='git://git.gnunet.org/gnunet-rpm.git'), 495 treeStableTimer=None, 496 builderNames=["gnunet_rpm_copr"])) 497 SCHEDULERS.append(schedulers.SingleBranchScheduler( 498 name="registrar_scheduler", 499 change_filter=util.ChangeFilter(branch='master', 500 repository='git://git.gnunet.org/gnunet-gns-registrar.git'), 501 treeStableTimer=None, 502 builderNames=["gnunet-gns-registrar"])) 503 504 505 # Schedule a coverity pass monthly 506 SCHEDULERS.append(schedulers.Nightly(name='nightly_cov', 507 change_filter=util.ChangeFilter(branch='master', 508 repository='git://git.gnunet.org/gnunet.git'), 509 builderNames=['gnunet_coverity'], 510 onlyIfChanged=True, 511 dayOfMonth=0, hour=3, minute=0)) 512 513 514 515 516 # 517 # WEBSITES 518 # 519 SCHEDULERS.append(schedulers.SingleBranchScheduler( 520 name="www_master_scheduler", 521 change_filter=util.ChangeFilter(branch='master', 522 repository='git://git.gnunet.org/www.git'), 523 treeStableTimer=None, 524 builderNames=["stage.gnunet.org"])) 525 526 SCHEDULERS.append(schedulers.SingleBranchScheduler( 527 name="www_stable_scheduler", 528 change_filter=util.ChangeFilter(branch='stable', 529 repository='git://git.gnunet.org/www.git'), 530 treeStableTimer=None, 531 builderNames=["www.gnunet.org"])) 532 SCHEDULERS.append(schedulers.SingleBranchScheduler( 533 name="bib_scheduler", 534 change_filter=util.ChangeFilter(branch='master', 535 repository='git://git.gnunet.org/gnunetbib.git'), 536 treeStableTimer=None, 537 builderNames=["bib.gnunet.org"])) 538 SCHEDULERS.append(schedulers.SingleBranchScheduler( 539 name="reclaim_www_scheduler", 540 change_filter=util.ChangeFilter(branch='master', 541 repository='git://git.gnunet.org/www-reclaim.git'), 542 treeStableTimer=None, 543 builderNames=["reclaim.gnunet.org"])) 544 SCHEDULERS.append(schedulers.SingleBranchScheduler( 545 name="rest_scheduler", 546 change_filter=util.ChangeFilter(branch='master', 547 repository='git://git.gnunet.org/gnunet-rest-api.git'), 548 treeStableTimer=None, 549 builderNames=["rest.gnunet.org"])) 550 SCHEDULERS.append(schedulers.SingleBranchScheduler( 551 name="lsd_scheduler", 552 change_filter=util.ChangeFilter(branch='master', 553 repository_re='git://git.gnunet.org/lsd.*'), 554 treeStableTimer=None, 555 builderNames=["lsd.gnunet.org"])) 556 SCHEDULERS.append(schedulers.SingleBranchScheduler( 557 name="gana_scheduler", 558 change_filter=util.ChangeFilter(branch='master', 559 repository='git://git.gnunet.org/gana.git'), 560 treeStableTimer=None, 561 builderNames=["gana.gnunet.org"])) 562 SCHEDULERS.append(schedulers.SingleBranchScheduler( 563 name="doc_scheduler", 564 change_filter=util.ChangeFilter(branch='master', 565 repository='git://git.gnunet.org/gnunet-handbook.git'), 566 treeStableTimer=None, 567 builderNames=["doc.gnunet.org"])) 568 569 570 # 571 # Buildbot self-reconfigure 572 # 573 SCHEDULERS.append(schedulers.SingleBranchScheduler( 574 name="bb_reload_scheduler", 575 change_filter=util.ChangeFilter(branch='master', 576 repository='git://git.gnunet.org/buildbot-ci.git'), 577 treeStableTimer=None, 578 builderNames=["buildbot"])) 579 580 581 582 def add_default_pipeline(bldr): 583 add_default_step(bldr, "build") 584 add_default_step(bldr, "install") 585 add_default_step(bldr, "test") 586 add_default_step(bldr, "deploy") 587 588 589 factory = util.BuildFactory() 590 591 592 ########################### 593 # GNUnet # 594 ########################### 595 gnunet_make_step = steps.ShellSequence( 596 name=util.Interpolate("GNUnet build"), 597 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 598 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"], 599 'TMPDIR': '/tmp/gnunet/'}, 600 haltOnFailure=True, 601 commands=[ 602 util.ShellArg(command=['./bootstrap'], logname='bootstrap', haltOnFailure=True), 603 util.ShellArg(command=['./configure', 604 "--prefix=/tmp/gnunet-buildbot", 605 "--enable-experimental", 606 "--enable-logging=verbose"], 607 logname='configure', 608 haltOnFailure=True), 609 util.ShellArg(command=['make'], 610 logname='make', 611 haltOnFailure=True), 612 util.ShellArg(command=['make', 'install'], 613 logname='make install', 614 haltOnFailure=True), 615 #util.ShellArg(command=['sudo', '/home/buildbot/bin/netjail.sh', 'bash', '-c', "'make check'"], 616 # logname='make check', 617 # warnOnFailure=True, 618 # flunkOnFailure=False), # make check has issues. 619 util.ShellArg(command=['make', 'uninstall'], 620 logname='make uninstall', 621 haltOnFailure=True)] 622 ) 623 624 gnunet_build_steps = [ 625 steps.Git(repourl='git://git.gnunet.org/gnunet.git', 626 mode='full', method='fresh'), 627 gnunet_make_step 628 ] 629 630 factory.addSteps(gnunet_build_steps) 631 632 ############################ 633 # COVERITY # 634 # Occurs: 1st day of month # 635 ############################ 636 cov_factory = util.BuildFactory() 637 cov_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='fresh')) 638 cov_factory.addStep(steps.ShellSequence( 639 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 640 env={'PATH': "${HOME}/.local/bin:${HOME}/bin:${PATH}"}, 641 commands=[ 642 util.ShellArg(command=['./bootstrap'], logname='bootstrap'), 643 util.ShellArg(command=['./configure', 644 "--prefix=/tmp/gnunet-buildbot", 645 "--enable-experimental=true"], 646 logname="configure"), 647 util.ShellArg(command=['cov-build', '--dir', 'cov-int', 'make'], logname='cov-build'), 648 ])) 649 cov_factory.addStep(steps.ShellCommand(command=['tar', 'czf', 'coverity.tar.gz', 'cov-int/'], 650 haltOnFailure=True, 651 name="Packing up")) 652 cov_factory.addStep(steps.ShellCommand(command=['curl', 653 '--form', util.Interpolate('token=%(secret:coverity_token)s'), 654 '--form', 'email=mschanzenbach@posteo.de', 655 '--form', 'version="git master"', 656 '--form', 'file=@./coverity.tar.gz', 657 '--form', 'description="Buildbot triggered build"', 658 'https://scan.coverity.com/builds?project=GNUnet%2Fgnunet'], 659 haltOnFailure=True, 660 name="Sending")) 661 662 663 ################################################# 664 # RELEASE BUILD # 665 # Occurs: # 666 # 1. Nightly # 667 # 2. Upon a pushed release tag (vX.Y.Z) # 668 # 3. With a commit message containing !tarball # 669 ################################################# 670 dist_factory = util.BuildFactory() 671 # https://github.com/buildbot/buildbot/issues/6539 change to "copy" when fixed 672 dist_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='clobber')) 673 674 dist_factory.addStep(steps.ShellSequence( 675 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 676 haltOnFailure=True, 677 commands=[ 678 util.ShellArg(command=['./bootstrap'], 679 logname='bootstrap', 680 haltOnFailure=True), 681 util.ShellArg(command=['./configure'], 682 logname='configure', 683 haltOnFailure=True), 684 util.ShellArg(command=['git', 'status'], 685 logname='status before dist', 686 haltOnFailure=True), 687 util.ShellArg(command=['make', 'dist'], 688 logname="dist", 689 haltOnFailure=True), 690 util.ShellArg(command=['make', 'doxygen'], 691 haltOnFailure=True, 692 logname="Doxygen") 693 ])) 694 695 # Get version number of tarball 696 cmdmeson = r'ls -1 build/meson-dist/gnunet-*.tar.gz | sed "s/build\/meson-dist\/gnunet-//" | sed "s/.tar.gz//" | tail -n1' 697 #cmd = 'git describe --tags | sed "s/^v//"' 698 dist_factory.addStep(steps.SetPropertyFromCommand(hideStepIf=False, 699 command=cmdmeson, 700 property='gnunet_meson_releasever', 701 name="Getting release version")) 702 dist_factory.addStep(steps.ShellCommand(command=["tar", 703 "xf", 704 util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 705 '-C', 'build'], 706 haltOnFailure=True, 707 name="Extracting tarball")) 708 709 # Make doxygen 710 # Try to build dist package 711 dist_factory.addStep(steps.ShellSequence( 712 workdir=util.Interpolate('build/build/gnunet-%(prop:gnunet_meson_releasever)s'), 713 name=util.Interpolate("GNUnet %(prop:gnunet_meson_releasever)s tarball build"), 714 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 715 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"]}, 716 commands=[ 717 util.ShellArg(command=['mkdir', '-p', '$TMPDIR'],logname='tmpdir',haltOnFailure=True), 718 util.ShellArg(command=['./configure', 719 "--prefix=/tmp/gnunet-buildbot", 720 "--enable-experimental=true", 721 "--enable-logging=verbose", 722 "--mesonbuilddir=tarball_build"], 723 logname='setup', 724 haltOnFailure=True), 725 util.ShellArg(command=['make'], 726 logname='compile', 727 haltOnFailure=True), 728 util.ShellArg(command=['make', 'install'], 729 logname='install', 730 haltOnFailure=True), 731 util.ShellArg(command=['make', 'uninstall'], 732 logname='uninstall', 733 haltOnFailure=True)] 734 )) 735 736 # Upload artifact to https://buildbot.gnunet.org/artifacts 737 dist_factory.addStep(steps.FileUpload(workersrc=util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 738 mode=0o644, 739 masterdest=util.Interpolate("~/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"), 740 url=util.Interpolate("https://buildbot.gnunet.org/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"))) 741 742 743 # Update doxygen (TODO skit on nightly?) 744 dist_factory.addStep(steps.ShellSequence( 745 name=util.Interpolate("Deploy Doxygen"), 746 workdir=util.Interpolate('build/doc'), 747 commands=[ 748 util.ShellArg(command=['chmod', '-R', 'ag+rX', '../doc'], 749 logname='Permissions', 750 haltOnFailure=True), 751 util.ShellArg(command=['rsync', '-a', '--delete', 752 '../doc/doxygen', 753 'handbook@firefly.gnunet.org:~/doc_deployment/'], 754 logname='Deploy', 755 haltOnFailure=True), 756 ])) 757 758 759 760 761 ########################### 762 # Fedora COPR build # 763 ########################### 764 copr_factory = util.BuildFactory() 765 copr_factory.addStep(steps.ShellCommand(command=["curl", 766 "-H", "Content-Type: application/json", 767 "--data", "{}", 768 "-X", "POST", 769 "https://copr.fedorainfracloud.org/webhooks/custom/36992/d316c169-1482-4508-a765-cfb5c0efeb67/gnunet/"], 770 haltOnFailure=True, 771 name="Triggering Copr build")) 772 773 774 ########################### 775 # gnunet-gns-registrar # 776 ########################### 777 registrar_factory = util.BuildFactory() 778 registrar_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-gns-registrar.git', mode='incremental')) 779 add_default_pipeline(registrar_factory) 780 781 ########################### 782 # stage.gnunet.org # 783 ########################### 784 www_factory = util.BuildFactory() 785 www_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental')) 786 add_default_pipeline(www_factory) 787 788 ########################### 789 # www.gnunet.org # 790 ########################### 791 www_stable_factory = util.BuildFactory() 792 www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental', branch='stable')) 793 add_default_pipeline(www_stable_factory) 794 795 ########################### 796 # bib.gnunet.org # 797 ########################### 798 bib_factory = util.BuildFactory() 799 bib_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunetbib.git', mode='incremental')) 800 add_default_pipeline(bib_factory) 801 802 803 ########################### 804 # reclaim.gnunet.org # 805 ########################### 806 reclaim_www_stable_factory = util.BuildFactory() 807 reclaim_www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www-reclaim.git', mode='incremental', branch='master')) 808 add_default_pipeline(reclaim_www_stable_factory) 809 810 811 ########################### 812 # rest.gnunet.org # 813 ########################### 814 rest_factory = util.BuildFactory() 815 rest_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-rest-api.git', mode='incremental', branch='master')) 816 add_default_pipeline(rest_factory) 817 818 ########################### 819 # lsd.gnunet.org # 820 ########################### 821 lsd_factory = util.BuildFactory() 822 lsd_factory.addStep(steps.Git(repourl=util.Property('repository'), mode='full', method="clobber", branch='master')) 823 add_default_pipeline(lsd_factory) 824 825 ########################### 826 # gana.gnunet.org # 827 ########################### 828 gana_factory = util.BuildFactory() 829 gana_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gana.git', mode='full', method="fresh", branch='master')) 830 add_default_pipeline(gana_factory) 831 832 ############################## 833 # doc.gnunet.org (Doc-NG) # 834 ############################## 835 doc_factory = util.BuildFactory() 836 doc_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-handbook.git', alwaysUseLatest=True, mode='full', method="clobber", branch='master')) 837 add_default_pipeline(doc_factory) 838 839 840 841 ########################### 842 # Buildbot TODO delete at some point # 843 ########################### 844 bb_factory = util.BuildFactory() 845 bb_factory.addStep(steps.Git(repourl='ssh://git@git.gnunet.org/buildbot-ci.git', mode='incremental')) 846 bb_factory.addStep(steps.ShellCommand(command=["./reload_bb-master.sh"], name="Reload configuration")) 847 848 849 850 BUILDERS.append( 851 util.BuilderConfig(name="gnunet-debian-x86_64", 852 workernames=["firefly-x86_64-amdepyc"], 853 factory=factory)) 854 855 BUILDERS.append( 856 util.BuilderConfig(name="gnunet-fedora-aarch64", 857 workernames=["schanzen-aarch64-fedora-meson"], 858 factory=factory)) 859 860 BUILDERS.append( 861 util.BuilderConfig(name="gnunet-debian-x86_64-dev", 862 workernames=["firefly-x86_64-amdepyc"], 863 factory=factory)) 864 865 BUILDERS.append( 866 util.BuilderConfig(name="gnunet-fedora-aarch64-dev", 867 workernames=["schanzen-aarch64-fedora-meson"], 868 factory=factory)) 869 870 871 BUILDERS.append( 872 util.BuilderConfig(name="gnunet_release", 873 workernames=["firefly-x86_64-amdepyc"], 874 factory=dist_factory)) 875 BUILDERS.append( 876 util.BuilderConfig(name="gnunet_coverity", 877 workernames=["firefly-x86_64-amdepyc"], 878 factory=cov_factory)) 879 880 BUILDERS.append( 881 util.BuilderConfig(name="gnunet-gns-registrar", 882 workernames=["firefly-x86_64-amdepyc", "schanzen-aarch64-fedora-meson"], 883 factory=registrar_factory)) 884 BUILDERS.append( 885 util.BuilderConfig(name="stage.gnunet.org", 886 workernames=["firefly-x86_64-amdepyc"], 887 factory=www_factory)) 888 BUILDERS.append( 889 util.BuilderConfig(name="www.gnunet.org", 890 workernames=["firefly-x86_64-amdepyc"], 891 factory=www_stable_factory)) 892 BUILDERS.append( 893 util.BuilderConfig(name="bib.gnunet.org", 894 workernames=["firefly-x86_64-amdepyc"], 895 factory=bib_factory)) 896 BUILDERS.append( 897 util.BuilderConfig(name="reclaim.gnunet.org", 898 workernames=["firefly-x86_64-amdepyc"], 899 factory=reclaim_www_stable_factory)) 900 BUILDERS.append( 901 util.BuilderConfig(name="rest.gnunet.org", 902 workernames=["firefly-x86_64-amdepyc"], 903 factory=rest_factory)) 904 BUILDERS.append( 905 util.BuilderConfig(name="lsd.gnunet.org", 906 workernames=["firefly-x86_64-amdepyc"], 907 factory=lsd_factory)) 908 BUILDERS.append( 909 util.BuilderConfig(name="gana.gnunet.org", 910 workernames=["firefly-x86_64-amdepyc"], 911 factory=gana_factory)) 912 BUILDERS.append( 913 util.BuilderConfig(name="doc.gnunet.org", 914 workernames=["firefly-x86_64-amdepyc"], 915 factory=doc_factory)) 916 BUILDERS.append( 917 util.BuilderConfig(name="buildbot", 918 workernames=["firefly-x86_64-amdepyc"], 919 factory=bb_factory)) 920 BUILDERS.append( 921 util.BuilderConfig(name="gnunet_rpm_copr", 922 workernames=["firefly-x86_64-amdepyc"], 923 factory=copr_factory)) 924 925 926 bsg = reporters.BuildStatusGenerator(mode=["exception", "failing", "problem"], 927 builders=["gnunet-debian-x86_64", "gnunet-fedora-aarch64"]) 928 929 mn = reporters.MailNotifier(fromaddr="buildbot@firefly.gnunet.org", 930 sendToInterestedUsers=False, 931 extraRecipients=["gnunet-ci@gnunet.org"], 932 lookup="gnunet.org", 933 generators=[bsg]) 934 SERVICES.append(mn) 935 936 937 938 939 ############################################# 940 # 19: CONTAINER FACTORY ##################### 941 ############################################# 942 ## 943 # These factories uses the standard container worker. 944 WORKERS.append(Worker("container-worker", "container-pass")) 945 946 947 # Container Job Generator Functions 948 # Parse config file and save values in a dict 949 def ingest_job_config(configPath, jobName): 950 configDict = {jobName: {}} 951 print(configDict) 952 ini.read_string(configPath) 953 for key in ini["build"]: 954 value = ini['build'][key] 955 configDict[jobName][key] = value 956 print(configDict) 957 configDict.update(configDict) 958 print(configDict) 959 return configDict 960 961 962 # Search for configs, and ingest 963 def handle_job_config(jobDirPath, jobName, repoName, configPath, configExist): 964 print(configPath) 965 if configExist == 0: 966 print(f"Ingesting Job Config: {configPath}") 967 configDict = ingest_job_config(configPath, jobName) 968 print(configDict) 969 return configDict 970 else: 971 print("No job config; Using default params") 972 # Set default job config parameters 973 configDict = {jobName: {"HALT_ON_FAILURE": True, 974 "WARN_ON_FAILURE": False, 975 "CONTAINER_BUILD": True, 976 "CONTAINER_NAME": repoName, 977 "CONTAINER_ARCH": "amd64"}} 978 return configDict 979 980 981 class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): 982 983 def __init__(self, REPO_NAME, **kwargs): 984 self.REPO_NAME = REPO_NAME 985 kwargs = self.setupShellMixin(kwargs) 986 super().__init__(**kwargs) 987 self.observer = logobserver.BufferLogObserver() 988 self.addLogObserver('stdio', self.observer) 989 990 def extract_stages(self, stdout): 991 stages = [] 992 for line in stdout.split('\n'): 993 stage = str(line.strip()) 994 if stage: 995 stages.append(stage) 996 return stages 997 998 @defer.inlineCallbacks 999 def run(self): 1000 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.REPO_NAME}" 1001 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1002 # run 'ls <project_root>/contrib/ci/jobs/' to get the list of stages 1003 cmd = yield self.makeRemoteShellCommand() 1004 yield self.runCommand(cmd) 1005 jobDirs = [] 1006 1007 # if the command passes extract the list of stages 1008 result = cmd.results() 1009 if result == util.SUCCESS: 1010 jobDirs = self.extract_stages(self.observer.getStdout()) 1011 print(f"this is jobDirs list: {jobDirs}") 1012 self.configDict = {} 1013 print(f"Remote cmd stdout: {self.observer.getStdout()}") 1014 print(f"cmd.results: {cmd.results()}") 1015 for stage in jobDirs: 1016 jobDirPath = f"{CI_JOBS_PATH}/{stage}" 1017 observer = logobserver.BufferLogObserver() 1018 self.addLogObserver('stdio', observer) 1019 cmd1 = yield self.makeRemoteShellCommand( 1020 command=["cat", f"{jobDirPath}/config.ini"]) 1021 yield self.runCommand(cmd1) 1022 print(f"cmd1.results: {cmd1.results()}") 1023 print(f"Second command stdout: {observer.getStdout()}") 1024 print(f"Current stage: {stage}") 1025 print(jobDirPath) 1026 self.configDict.update( 1027 handle_job_config( 1028 jobDirPath, stage, self.REPO_NAME, 1029 observer.getStdout(), cmd1.results())) 1030 print(self.configDict) 1031 # create a container step for each stage and 1032 # add them to the build 1033 convstr2bool = ast.literal_eval 1034 self.build.addStepsAfterCurrentStep([ 1035 container_add_step( 1036 convstr2bool( 1037 str(self.configDict[stage]["HALT_ON_FAILURE"])), 1038 convstr2bool( 1039 str(self.configDict[stage]["WARN_ON_FAILURE"])), 1040 convstr2bool( 1041 str(self.configDict[stage]["CONTAINER_BUILD"])), 1042 self.configDict[stage]["CONTAINER_NAME"], 1043 container_factory, 1044 CONTAINER_WORKDIR, 1045 self.REPO_NAME, 1046 stage, 1047 self.configDict[stage]["CONTAINER_ARCH"], 1048 f"contrib/ci/jobs/{stage}/job.sh") 1049 for stage in jobDirs 1050 ]) 1051 1052 return result 1053 1054 # List of repos to add to container factory. 1055 container_repos = ["git.gnunet.org/gnunet", 1056 "git.taler.net/challenger", 1057 "git.taler.net/exchange", 1058 "git.taler.net/libeufin", 1059 "git.taler.net/taler-rust", 1060 "git.taler.net/depolymerization", 1061 "git.taler.net/merchant", 1062 "git.taler.net/sandcastle-ng", 1063 "git.taler.net/sync", 1064 "git.taler.net/taler-android", 1065 "git.taler.net/taler-mailbox", 1066 "git.taler.net/taldir", 1067 "git.taler.net/taler-typescript-core",] 1068 1069 for repo in container_repos: 1070 1071 # Prepare to read job configs 1072 ini = configparser.ConfigParser() 1073 ini.optionxform = str 1074 1075 # Factory-wide variables 1076 REPO_NAME = repo.rsplit('/', 1)[1] 1077 REPO_URL = "git://" + repo + ".git" 1078 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{REPO_NAME}" 1079 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1080 1081 # Create a factory 1082 container_factory = util.BuildFactory() 1083 container_factory.workdir = CONTAINER_WORKDIR 1084 1085 # Setup workspace 1086 container_factory.addStep(ShellCommand( 1087 name="workspace", 1088 descriptionDone="Workspace directory check", 1089 command=f"test -d {CONTAINER_WORKDIR} && podman run --log-driver=none --rm --volume {CONTAINER_WORKDIR}:/workdir docker.io/library/debian:bookworm-slim chmod -R 777 /workdir ; rm -rf {CONTAINER_WORKDIR} && mkdir -p {CONTAINER_WORKDIR} || mkdir -p {CONTAINER_WORKDIR}", 1090 haltOnFailure=True, 1091 )) 1092 1093 # Ensure repo is cloned or clean. 1094 # Git() will clone repo if it doesn't exist. 1095 # Method clobber removes directory and makes a fresh clone. 1096 # Shallow set to "True" defaults to a depth of 1. 1097 # Will checkout value of "branch" property from job properties. 1098 # https://docs.buildbot.net/latest/manual/configuration/steps/source_git.html 1099 container_factory.addStep(Git( 1100 name="git", 1101 repourl=REPO_URL, 1102 branch=util.Interpolate('%(src::branch)s'), 1103 mode='full', 1104 method='clobber', 1105 shallow=True, 1106 submodules=True, 1107 haltOnFailure=True, 1108 )) 1109 1110 container_factory.addStep(GenerateStagesCommand( 1111 REPO_NAME, 1112 name="Generate build stages", 1113 command=f"ls {CI_JOBS_PATH}", 1114 haltOnFailure=True)) 1115 1116 BUILDERS.append(util.BuilderConfig( 1117 name=f"{REPO_NAME}-builder", 1118 workernames=["container-worker"], 1119 factory=container_factory 1120 )) 1121 1122 # Only enable this scheduler for debugging! 1123 # Will run builders with 1 minute of waiting inbetween builds 1124 # SCHEDULERS.append(schedulers.Periodic( 1125 # name=f"{REPO_NAME}-minutely", 1126 # builderNames=[f"{REPO_NAME}-builder"], 1127 # periodicBuildTimer=60 1128 # )) 1129 1130 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1131 name=f"{REPO_NAME}-container-scheduler", 1132 change_filter=util.ChangeFilter( 1133 branch="master", 1134 project_re=f"({REPO_NAME})" 1135 ), 1136 treeStableTimer=30, 1137 builderNames=[f"{REPO_NAME}-builder"] 1138 )) 1139 1140 SERVICES.append(reporters.MailNotifier( 1141 fromaddr="buildbot@taler.net", 1142 # notify from pass to fail, and viceversa. 1143 generators=[BuildStatusGenerator( 1144 mode=('change','problem','failing','exception',), 1145 builders=[f"{REPO_NAME}-builder",], 1146 message_formatter=reporters.MessageFormatter( 1147 template_type='plain', 1148 want_logs_content=True, 1149 ), 1150 )], 1151 sendToInterestedUsers=False, 1152 useTls=False, 1153 relayhost="localhost", 1154 smtpPort=25, 1155 dumpMailsToLog=True, 1156 extraRecipients=[f"ci-{REPO_NAME}@taler.net"] 1157 )) 1158 1159 1160 ############## sandcastle-ng Scheduler ################################# 1161 1162 1163 # Periodic scheduler for sandcastle-ng. 1164 # Runs every 2 hours (60 seconds * 60 * 2) 1165 SCHEDULERS.append(schedulers.Periodic( 1166 name="sandcastle-ng-periodic-scheduler", 1167 builderNames=["sandcastle-ng-builder"], 1168 change_filter=util.ChangeFilter(branch="master"), 1169 periodicBuildTimer=60*60*2 1170 )) 1171 1172 1173 ################ 99: debug stuff JOB ################################### 1174 1175 # This does nothing, just a starting point for a factory. 1176 DEBUG_FACTORY = util.BuildFactory() 1177 DEBUG_FACTORY.addStep( 1178 ShellCommand( 1179 name="echo debug", 1180 description="just echoing a word", 1181 descriptionDone="builder responded", 1182 command=["echo", "I'm here!"] 1183 ) 1184 ) 1185 1186 1187 ################################################################## 1188 #################### General purpose ############################# 1189 ################################################################## 1190 1191 # Compute array of the names of all of our builders 1192 BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) 1193 1194 ####### GENERAL PURPOSE BUILDBOT SERVICES ####################### 1195 1196 SERVICES.append(reporters.MailNotifier( 1197 fromaddr="testbuild@taler.net", 1198 # notify from pass to fail, and viceversa. 1199 generators=[BuildStatusGenerator( 1200 mode=('change','problem','failing','exception',), 1201 builders=EMAIL_ALERTS, 1202 message_formatter=reporters.MessageFormatter( 1203 template_type='plain', 1204 want_logs_content=True, 1205 ), 1206 )], 1207 sendToInterestedUsers=False, 1208 useTls=False, 1209 relayhost="localhost", 1210 smtpPort=25, 1211 dumpMailsToLog=True, 1212 extraRecipients=BUILDER_EMAIL_ADDRESSES 1213 )) 1214 1215 1216 ############# GENERAL PURPOSE SCHEDULERS ########################## 1217 1218 # Workers that are done on wallet or deployment changes to master 1219 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1220 name="taler-healthcheck-scheduler", 1221 change_filter=util.ChangeFilter( 1222 branch="master", 1223 project_re="(taler-typescript-core|taler-deployment)" 1224 ), 1225 treeStableTimer=None, 1226 builderNames=WALLETCHANGE_TRIGGERS 1227 )) 1228 1229 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1230 name="all-scheduler", 1231 change_filter=util.ChangeFilter( 1232 branch_re="(master|stable)", 1233 project_re="(taler-typescript-core|exchange|" 1234 "merchant|taler-deployment|twister|sync|" 1235 "taler-merchant-demos)" 1236 ), 1237 treeStableTimer=None, 1238 builderNames=CODECHANGE_TRIGGERS 1239 )) 1240 1241 # Scheduler for all nightly builds. 1242 SCHEDULERS.append(schedulers.Nightly( 1243 name="nightly-scheduler", 1244 builderNames=list(NIGHTLY_TRIGGERS), 1245 branch="master", 1246 hour=6, 1247 minute=0 1248 )) 1249 1250 # Provide "force" button in the web UI. 1251 SCHEDULERS.append(schedulers.ForceScheduler( 1252 name="force-scheduler", 1253 buttonName="Force build", 1254 builderNames=list(BUILDER_LIST) 1255 )) 1256 1257 ######################################################### 1258 ####### Actual configuation initialization ############## 1259 ######################################################### 1260 1261 # This is the dictionary that the buildmaster pays attention to. We also use 1262 # a shorter alias to save typing. 1263 c = BuildmasterConfig = {} 1264 1265 # Secrets 1266 c['secretsProviders'] = [secrets.SecretInAFile(dirname="/home/buildbot-master/secrets")] 1267 1268 c["workers"] = WORKERS 1269 c["builders"] = BUILDERS 1270 c["schedulers"] = SCHEDULERS 1271 c["services"] = SERVICES 1272 1273 # Silence warning and allow very basic phoning home. 1274 c["buildbotNetUsageData"] = "basic" 1275 1276 c["title"] = "GNUnet Builder" 1277 c["titleURL"] = "https://buildbot.gnunet.org" 1278 1279 # the 'buildbotURL' string should point to the location where the buildbot's 1280 # internal web server is visible. 1281 c['buildbotURL'] = "https://buildbot.gnunet.org/" 1282 1283 # This specifies what database buildbot uses to store its 1284 # state. You can leave this at its default for all but the 1285 # largest installations. 1286 c["db"] = { 1287 "db_url": "sqlite:///state.sqlite", 1288 } 1289 1290 # the 'change_source' setting tells the buildmaster how it should 1291 # find out about source code changes. 1292 pbSource = changes.PBChangeSource(port="tcp:19990:interface=127.0.0.1", 1293 user="git-buildbot", 1294 passwd="Aer3eari") 1295 1296 pollGnunetSource = changes.GitPoller(repourl='git://git.gnunet.org/gnunet.git', 1297 branches=True, 1298 pollInterval=300, 1299 pollAtLaunch=True, 1300 project="gnunet") 1301 c["change_source"] = [pollGnunetSource, pbSource] 1302 1303 # 'protocols' contains information about protocols which master 1304 # will use for communicating with workers. You must define at 1305 # least 'port' option that workers could connect to your master 1306 # with this protocol. 'port' must match the value configured into 1307 # the workers (with their --master option) 1308 c["protocols"] = {"pb": {"port": "tcp:19989"}} 1309 1310 # Load admin password 1311 with open("/home/buildbot-master/secrets/admin") as fh: 1312 www_pass = fh.read().strip() 1313 1314 # minimalistic config to activate new web UI 1315 # -- formerly commented out as not packaged properly in Debian and others, see 1316 # https://bugzilla.redhat.com/show_bug.cgi?id=1557687 1317 c["www"] = { 1318 "port": "tcp:8009:interface=127.0.0.1", 1319 "default_page": 'builders', 1320 "plugins": { 1321 "waterfall_view": True, 1322 "console_view": True, 1323 "grid_view": True, 1324 }, 1325 "auth": util.UserPasswordAuth([('admin',www_pass)]), 1326 "allowed_origins": ["https://*.taler.net","https://*.gnunet.org"], 1327 "avatar_methods": [], 1328 }