master.cfg (55686B)
1 # -*- python -*- 2 # ex: set syntax=python: 3 4 ## 5 # This file is part of TALER 6 # (C) 2016-2025 Taler Systems SA 7 # 8 # TALER is free software; you can redistribute it and/or 9 # modify it under the terms of the GNU Affero General Public 10 # License as published by the Free Software Foundation; either 11 # version 3, or (at your option) any later version. 12 # 13 # TALER is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty 15 # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 16 # See the GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public 19 # License along with TALER; see the file COPYING. If not, 20 # see <http://www.gnu.org/licenses/> 21 # 22 # @author Florian Dold 23 # @author Marcello Stanisci 24 # @author ng0 25 # @author Christian Grothoff 26 # @author Devan Carpenter 27 import ast 28 import configparser 29 import glob 30 import os 31 import pathlib 32 import pwd 33 import re 34 import subprocess 35 36 from buildbot.changes.pb import PBChangeSource 37 from buildbot.steps.source.git import Git 38 from buildbot.steps.shell import ShellCommand 39 from buildbot.plugins import changes 40 from buildbot.plugins import reporters 41 from buildbot.plugins import schedulers 42 from buildbot.plugins import steps 43 from buildbot.plugins import secrets, util 44 from buildbot.process import buildstep, logobserver 45 from buildbot.process.results import SKIPPED 46 from buildbot.reporters.generators.build import BuildStatusGenerator 47 from buildbot.worker import Worker 48 from twisted.internet import defer 49 50 # This is a sample buildmaster config file. It must be 51 # installed as 'master.cfg' in your buildmaster's base 52 # directory. 53 54 # This file has the following structure: 55 # - Globals: definition of global variables we use throughout 56 # + Convenience functions: helper functions useful for many jobs 57 # - Jobs: actual job definitions 58 # - General purpose: triggers and alerts shared by various jobs 59 # + general purpose notification (alerts) 60 # + general purpose triggers (schedulers) 61 # - Actual buildbot configuration object initialization 62 63 ################################################################# 64 ######################### GLOBALS ############################### 65 ################################################################# 66 67 # The 'workers' list defines the set of recognized workers. 68 # Each element is a Worker object, specifying a unique worker 69 # name and password. The same worker name and password must 70 # be configured on the worker. 71 WORKERS = [] 72 73 # 'services' is a list of BuildbotService items like reporter 74 # targets. The status of each build will be pushed to these 75 # targets. buildbot/reporters/*.py has a variety to choose from, 76 # like IRC bots. 77 78 79 class MessageFormatterWithStdout(reporters.MessageFormatter): 80 def buildAdditionalContext(self, master, ctx): 81 stdout = [] 82 for step in ctx["build"]["steps"]: 83 for log in step["logs"]: 84 all_logs = log["content"]["content"].splitlines() 85 # Including only what the script printed on stdout. 86 for line in all_logs: 87 if re.search("^o", line): 88 stdout.append(line[1:]) 89 ctx.update(dict(stdout="\n".join(stdout))) 90 91 92 ##################################################### 93 # Commit message triggers # 94 # # 95 # This checks for triggers in the commit messages # 96 # !tarball will trigger a release build # 97 # !coverity will trigger a coverity check # 98 ##################################################### 99 def checkForTarballTrigger(change): 100 if "!tarball" in change.comments: 101 return True 102 return False 103 def checkForCoverityTrigger(change): 104 if "!coverity" in change.comments: 105 return True 106 107 SERVICES = [] 108 109 # The 'builders' list defines the Builders, which tell Buildbot 110 # how to perform a build: what steps, and which workers can execute 111 # them. Note that any particular build will only take place on 112 # one worker. 113 BUILDERS = [] 114 115 # Configures the Schedulers, which decide how to react to incoming 116 # changes. 117 SCHEDULERS = [] 118 119 # Array of builders to be scheduled every night. 120 NIGHTLY_TRIGGERS=[] 121 122 # Array of builders to be scheduled whenever any of the code Git repos change 123 CODECHANGE_TRIGGERS = [] 124 125 # Array of builders to be scheduled whenever the taler-typescript-core or 126 # taler-deployment change 127 WALLETCHANGE_TRIGGERS = [] 128 129 # Array of builder names for which build status reports should be sent 130 # via e-mail 131 EMAIL_ALERTS = [] 132 133 # Array of email address for which build status reports shoudl be sent 134 BUILDER_EMAIL_ADDRESSES = [] 135 136 ############ Convenience functions ################# 137 138 # Create a FACTORY with a taler-deployment.git checkout as the first step. 139 def create_factory_with_deployment(): 140 f = util.BuildFactory() 141 update_deployment(f) 142 return f 143 144 # Convenience function that checks out a Git repository. 145 # First argument is the URL of the Git to clone, second 146 # the desired branch. Default is 'master'. 147 def git_step(repo, target_branch="master"): 148 return Git( 149 repourl=repo, 150 mode="full", 151 method="fresh", 152 logEnviron=False, 153 alwaysUseLatest=True, 154 haltOnFailure=True, 155 branch=target_branch 156 ) 157 158 # FIXME: document this function! 159 def add_default_step(bldr, step): 160 worker_script = ".buildbot/%(prop:workername)s_" + step + ".sh" 161 default_script = ".buildbot/" + step + ".sh" 162 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (worker_script,worker_script) 163 bldr.addStep(steps.SetPropertyFromCommand(command=util.Interpolate(cmd), 164 hideStepIf=True, 165 property='buildbotScript_%s' % step, 166 name="Checking for worker-specific %s script" % step)) 167 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (default_script,default_script) 168 bldr.addStep(steps.SetPropertyFromCommand(doStepIf=(util.Property('buildbotScript_%s' % step) != util.Interpolate(worker_script)), 169 hideStepIf=True, 170 command=util.Interpolate(cmd), 171 property='buildbotScript_%s' % step, 172 name="Checking for %s script" % step)) 173 bldr.addStep(steps.ShellCommand(command=util.Property('buildbotScript_%s' % step), 174 haltOnFailure=True, 175 env={'PATH': "${HOME}/.local/bin:$HOME/bin:${PATH}"}, 176 doStepIf=(util.Property('buildbotScript_%s' % step) != None), 177 hideStepIf=lambda results, s: results==SKIPPED, 178 name="Executing %s step" % step)) 179 180 181 182 # Convenience function that runs 'make check' in a 183 # directory of the code inside of a netjail. 184 def jailed_check(package, srcdirs): 185 return steps.ShellSequence( 186 name="Tests of " + package, 187 description="Testing " + package, 188 descriptionDone="Pass", 189 commands=map(lambda srcdir: util.ShellArg(command=["sudo", "/usr/local/bin/netjail.sh", "/home/container-worker/taler-deployment/buildbot/with-postgres.sh", "bash", "-c", "'cd src/"+srcdir+" make install check'"]), srcdirs), 190 env={'PATH': "${HOME}/local/bin:${PATH}"}, 191 workdir="../../sources/" + package 192 ) 193 194 # Convenience function that checks out the deployment. 195 def update_deployment(factory): 196 factory.addStep(steps.ShellSequence( 197 name="update taler-deployment", 198 description="removing old deployment and fetching fresh repository", 199 descriptionDone="Deployment updated", 200 commands=[ 201 util.ShellArg(command=["rm", "-rf", "taler-deployment"]), 202 util.ShellArg(command=["git", "clone", "git://git.taler.net/taler-deployment"]), 203 ], 204 haltOnFailure=True, 205 workdir="../.." 206 )) 207 208 # Convenience function that builds and runs a container. 209 def container_add_step(HALT_ON_FAILURE, 210 WARN_ON_FAILURE, 211 CONTAINER_BUILD, 212 CONTAINER_NAME, 213 factory, 214 WORK_DIR, 215 repoName, 216 stepName, 217 CONTAINER_ARCH="amd64", 218 jobCmd="/workdir/contrib/ci/ci.sh", 219 containerFile="contrib/ci/Containerfile"): 220 print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") 221 222 # Locally built images belong to this repository, job and architecture. 223 # A copied CONTAINER_NAME must not overwrite another builder's image 224 # between its build and run commands on the shared container worker. 225 # CONTAINER_NAME selects the image only for jobs using prebuilt images. 226 imageName = (f"localhost/buildbot/{repoName}/{stepName}:{CONTAINER_ARCH}" 227 if CONTAINER_BUILD else CONTAINER_NAME) 228 229 runCommand = ["podman", "--transient-store", "run", "--rm", 230 "--arch", CONTAINER_ARCH, 231 "--log-driver=none", 232 "--add-host", "taler.host.internal:10.0.2.2", 233 "--network", "slirp4netns:allow_host_loopback=true", 234 "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), 235 "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), 236 "--env", util.Interpolate("CI_PROJECT_NAME=%(src::project)s"), 237 "--cap-add", "SYS_ADMIN,CAP_SYS_CHROOT", 238 "--volume", f"{WORK_DIR}:/workdir", 239 "--volume", "/home/container-worker/ephemeral_ci_artifacts:/artifacts", 240 "--volume", "/home/container-worker/persistent_ci_artifacts:/persistent_artifacts", 241 "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", 242 "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", 243 "--workdir", "/workdir"] 244 245 # Inputs directory 246 inputs_path = f"/home/container-worker/container_inputs/{repoName}" 247 print(f"Checking that {inputs_path} exists") 248 if os.path.isdir(inputs_path): 249 print(f"Adding {inputs_path}") 250 runCommand += ["--volume", f"{inputs_path}:/inputs:ro"] 251 else: 252 print(f"Inputs directory not found at {inputs_path}") 253 254 if CONTAINER_BUILD: 255 runCommand += ["--volume", f"/run/user/{pwd.getpwnam('container-worker').pw_uid}/podman/podman.sock:/run/podman/podman.sock", 256 "--security-opt", "label=disable"] 257 258 runCommand += [imageName, jobCmd] 259 260 # Halting jobs (including deployment) remain fatal even if they also 261 # request warnings. Only non-halting jobs may be advisory. 262 advisory = WARN_ON_FAILURE and not HALT_ON_FAILURE 263 runArg = util.ShellArg(command=runCommand, 264 logname='run inside container', 265 flunkOnFailure=not advisory, 266 warnOnFailure=advisory, 267 haltOnFailure=HALT_ON_FAILURE) 268 269 buildArg = util.ShellArg(command=["podman", "build", "-t", imageName, 270 "--arch", CONTAINER_ARCH, 271 "-f", containerFile, "."], 272 logname='build container', haltOnFailure=True) 273 274 if not CONTAINER_BUILD: 275 return steps.ShellSequence( 276 name=stepName, 277 commands=[runArg], 278 haltOnFailure=HALT_ON_FAILURE, 279 flunkOnFailure=True, 280 warnOnWarnings=True, 281 workdir=WORK_DIR 282 ) 283 else: 284 return steps.ShellSequence( 285 name=stepName, 286 commands=[buildArg, runArg], 287 haltOnFailure=HALT_ON_FAILURE, 288 flunkOnFailure=True, 289 warnOnWarnings=True, 290 workdir=WORK_DIR 291 ) 292 293 ################################################################## 294 ######################## JOBS #################################### 295 ################################################################## 296 297 # For every job, we have (in this order!): 298 # - worker(s): hosts/users that run the job 299 # - factory: list of steps that define what to do 300 # - builder: gives the job a name and binds it to the factory and worker 301 # - (OPTIONAL) alerts: notifications to trigger when the job fails 302 # Pre-defined: EMAIL_ALERTS 303 # - scheduler: rules that define when to run the job 304 # Pre-defined: NIGHTLY_TRIGGERS, CODECHANGE_TRIGGERS, WALLETCHANGE_TRIGGERS 305 306 ################ 1: BUILDMASTER JOB ################################### 307 308 ## 309 # This worker restarts the buildmaster itself on 310 # changes to this file. 311 # Location: /home/buildbot-master 312 WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) 313 314 BUILDMASTER_FACTORY = create_factory_with_deployment() 315 BUILDMASTER_FACTORY.addStep( 316 ShellCommand( 317 name="restart buildmaster", 318 description="trigger buildmaster restart with new configuration", 319 descriptionDone="Buildmaster updated", 320 command=["pkill", "-HUP", "-A", "-u", "buildbot-master", "-f", "/usr/bin/python3"], 321 workdir="../.." 322 ) 323 ) 324 325 BUILDERS.append(util.BuilderConfig( 326 name="buildmaster-builder", 327 workernames=["buildmaster-worker"], 328 factory=BUILDMASTER_FACTORY 329 )) 330 331 EMAIL_ALERTS.append("buildmaster-builder") 332 333 # Buildmaster is notified whenever deployment.git changes 334 SCHEDULERS.append(schedulers.SingleBranchScheduler( 335 name="buildmaster-scheduler", 336 change_filter=util.ChangeFilter( 337 branch="master", 338 project_re="(taler-deployment)" 339 ), 340 treeStableTimer=None, 341 builderNames=["buildmaster-builder"] 342 )) 343 344 ################ 2: WEBSITE JOB ################################### 345 346 ## 347 # This worker builds Websites: www and stage. 348 # 349 WORKERS.append(Worker("sites-worker", "sites-pass")) 350 351 SITES_FACTORY = create_factory_with_deployment() 352 SITES_FACTORY.addStep( 353 ShellCommand( 354 name="build Web sites", 355 description="Building all the Taler homepages", 356 descriptionDone="Sites built.", 357 command=["./build-sites.sh"], 358 workdir="../../taler-deployment/worker-sites", 359 haltOnFailure=True 360 ) 361 ) 362 363 BUILDERS.append(util.BuilderConfig( 364 name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY 365 )) 366 367 #EMAIL_ALERTS.append("sites-builder") 368 369 370 # The web page changed if 'taler-www', 'taler-tutorials', 371 # 'taler-docs' or 'twister' changed 372 def web_page(change): 373 _change = change.asDict() 374 repo = _change.get("project") 375 if repo in ["taler-docs", "taler-tutorials", "taler-www", "twister", "taler-deployment", "buywith", "taler-ops-www", "taler-systems-www", "anastasis-www", "pepsi"]: 376 return True 377 return False 378 379 380 # Sites are re-build whenever taler-deployment, taler-www, or twister changes. 381 SCHEDULERS.append(schedulers.SingleBranchScheduler( 382 name="sites-scheduler", 383 builderNames=["sites-builder"], 384 change_filter=util.ChangeFilter( 385 branch_re="(master|stable)", 386 filter_fn=web_page 387 ), 388 treeStableTimer=None 389 )) 390 391 392 ################ 3: 'check links' JOB ################################### 393 394 ## 395 # linkchecker worker checks for dead links in the Website 396 # Location: /home/linkchecker-worker 397 WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) 398 399 # linkchecker FACTORY 400 LINKCHECKER_FACTORY = create_factory_with_deployment() 401 LINKCHECKER_FACTORY.addStep( 402 ShellCommand( 403 name="linkchecker", 404 description="Check taler.net website for broken links && Notify", 405 descriptionDone="Results of wget in buildbot logs.", 406 command=["/home/linkchecker-worker/taler-deployment/worker-linkchecker/linkchecker.sh"], 407 workdir="/home/linkchecker-worker", 408 haltOnFailure=True, 409 timeout=7200 # 2 hours 410 ) 411 ) 412 413 # linkchecker BUILDER 414 # worker at linkchecker@taler.net 415 BUILDERS.append(util.BuilderConfig( 416 name="linkchecker-builder", 417 workernames="linkchecker-worker", 418 factory=LINKCHECKER_FACTORY 419 )) 420 421 docs_generator = BuildStatusGenerator( 422 mode=('change', 'problem', 'failing', 'exception',), 423 builders=[ 424 'linkchecker-builder', 425 ], 426 message_formatter=reporters.MessageFormatter( 427 template_type='plain', 428 want_logs_content=True 429 ) 430 ) 431 432 433 SERVICES.append(reporters.MailNotifier( 434 fromaddr="bb@taler.net", 435 generators=[docs_generator], 436 sendToInterestedUsers=False, 437 useTls=False, 438 relayhost="localhost", 439 smtpPort=25, 440 dumpMailsToLog=True, 441 extraRecipients=['linkcheck@taler.net'] 442 )) 443 444 # SERVICES.append(tipReserveEmails) 445 446 NIGHTLY_TRIGGERS.append("linkchecker-builder") 447 448 #################### 449 ## GNUnet workers ## 450 #################### 451 452 WORKERS.append(Worker("firefly-x86_64-amdepyc", "pass")) 453 WORKERS.append(Worker("mp-amd64-openbsd", "Tai7zeic")) 454 WORKERS.append(Worker("schanzen-aarch64-fedora-meson", "eWi9keet")) 455 456 457 SCHEDULERS.append(schedulers.AnyBranchScheduler( 458 name="gnunet", 459 change_filter=util.ChangeFilter(branch_re='master', 460 repository='git://git.gnunet.org/gnunet.git'), 461 treeStableTimer=None, 462 builderNames=["gnunet-debian-x86_64", 463 "gnunet-fedora-aarch64"])) 464 465 SCHEDULERS.append(schedulers.AnyBranchScheduler( 466 name="gnunet-dev", 467 change_filter=util.ChangeFilter(branch_re='dev/.+', 468 repository='git://git.gnunet.org/gnunet.git'), 469 treeStableTimer=None, 470 builderNames=["gnunet-debian-x86_64-dev", 471 "gnunet-fedora-aarch64-dev"])) 472 473 SCHEDULERS.append(schedulers.SingleBranchScheduler( 474 name="tagged_release", 475 change_filter=util.ChangeFilter(branch_re='.*v[0-9]*[.][0-9]*[.][0-9]*$', 476 repository='git://git.gnunet.org/gnunet.git'), 477 treeStableTimer=None, 478 builderNames=["gnunet_release"])) 479 480 SCHEDULERS.append(schedulers.SingleBranchScheduler( 481 name="tarball", 482 onlyImportant=True, 483 change_filter=util.ChangeFilter(branch='master', 484 repository='git://git.gnunet.org/gnunet.git'), 485 fileIsImportant=checkForTarballTrigger, 486 treeStableTimer=None, 487 builderNames=["gnunet_release"])) 488 489 490 # Build a tarball nightly 491 SCHEDULERS.append(schedulers.Nightly(name='nightly', 492 change_filter=util.ChangeFilter(branch='master', 493 repository='git://git.gnunet.org/gnunet.git'), 494 builderNames=['gnunet_release'], 495 onlyIfChanged=True, 496 hour=6, minute=0)) 497 498 SCHEDULERS.append(schedulers.SingleBranchScheduler( 499 name="gnunet_cov", 500 onlyImportant=True, 501 change_filter=util.ChangeFilter(branch='master', 502 repository='git://git.gnunet.org/gnunet.git'), 503 fileIsImportant=checkForCoverityTrigger, 504 treeStableTimer=None, 505 builderNames=["gnunet_coverity"])) 506 507 SCHEDULERS.append(schedulers.SingleBranchScheduler( 508 name="gnunet_rpm", 509 change_filter=util.ChangeFilter(branch='dev/schanzen/copr', 510 repository='git://git.gnunet.org/gnunet-rpm.git'), 511 treeStableTimer=None, 512 builderNames=["gnunet_rpm_copr"])) 513 SCHEDULERS.append(schedulers.SingleBranchScheduler( 514 name="registrar_scheduler", 515 change_filter=util.ChangeFilter(branch='master', 516 repository='git://git.gnunet.org/gnunet-gns-registrar.git'), 517 treeStableTimer=None, 518 builderNames=["gnunet-gns-registrar"])) 519 520 521 # Schedule a coverity pass monthly 522 SCHEDULERS.append(schedulers.Nightly(name='nightly_cov', 523 change_filter=util.ChangeFilter(branch='master', 524 repository='git://git.gnunet.org/gnunet.git'), 525 builderNames=['gnunet_coverity'], 526 onlyIfChanged=True, 527 dayOfMonth=0, hour=3, minute=0)) 528 529 530 531 532 # 533 # WEBSITES 534 # 535 SCHEDULERS.append(schedulers.SingleBranchScheduler( 536 name="www_master_scheduler", 537 change_filter=util.ChangeFilter(branch='master', 538 repository='git://git.gnunet.org/www.git'), 539 treeStableTimer=None, 540 builderNames=["stage.gnunet.org"])) 541 542 SCHEDULERS.append(schedulers.SingleBranchScheduler( 543 name="www_stable_scheduler", 544 change_filter=util.ChangeFilter(branch='stable', 545 repository='git://git.gnunet.org/www.git'), 546 treeStableTimer=None, 547 builderNames=["www.gnunet.org"])) 548 SCHEDULERS.append(schedulers.SingleBranchScheduler( 549 name="bib_scheduler", 550 change_filter=util.ChangeFilter(branch='master', 551 repository='git://git.gnunet.org/gnunetbib.git'), 552 treeStableTimer=None, 553 builderNames=["bib.gnunet.org"])) 554 SCHEDULERS.append(schedulers.SingleBranchScheduler( 555 name="reclaim_www_scheduler", 556 change_filter=util.ChangeFilter(branch='master', 557 repository='git://git.gnunet.org/www-reclaim.git'), 558 treeStableTimer=None, 559 builderNames=["reclaim.gnunet.org"])) 560 SCHEDULERS.append(schedulers.SingleBranchScheduler( 561 name="rest_scheduler", 562 change_filter=util.ChangeFilter(branch='master', 563 repository='git://git.gnunet.org/gnunet-rest-api.git'), 564 treeStableTimer=None, 565 builderNames=["rest.gnunet.org"])) 566 SCHEDULERS.append(schedulers.SingleBranchScheduler( 567 name="lsd_scheduler", 568 change_filter=util.ChangeFilter(branch='master', 569 repository_re='git://git.gnunet.org/lsd.*'), 570 treeStableTimer=None, 571 builderNames=["lsd.gnunet.org"])) 572 SCHEDULERS.append(schedulers.SingleBranchScheduler( 573 name="gana_scheduler", 574 change_filter=util.ChangeFilter(branch='master', 575 repository='git://git.gnunet.org/gana.git'), 576 treeStableTimer=None, 577 builderNames=["gana.gnunet.org"])) 578 SCHEDULERS.append(schedulers.SingleBranchScheduler( 579 name="doc_scheduler", 580 change_filter=util.ChangeFilter(branch='master', 581 repository='git://git.gnunet.org/gnunet-handbook.git'), 582 treeStableTimer=None, 583 builderNames=["doc.gnunet.org"])) 584 585 586 # 587 # Buildbot self-reconfigure 588 # 589 SCHEDULERS.append(schedulers.SingleBranchScheduler( 590 name="bb_reload_scheduler", 591 change_filter=util.ChangeFilter(branch='master', 592 repository='git://git.gnunet.org/buildbot-ci.git'), 593 treeStableTimer=None, 594 builderNames=["buildbot"])) 595 596 597 598 def add_default_pipeline(bldr): 599 add_default_step(bldr, "build") 600 add_default_step(bldr, "install") 601 add_default_step(bldr, "test") 602 add_default_step(bldr, "deploy") 603 604 605 factory = util.BuildFactory() 606 607 608 ########################### 609 # GNUnet # 610 ########################### 611 gnunet_make_step = steps.ShellSequence( 612 name=util.Interpolate("GNUnet build"), 613 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 614 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"], 615 'TMPDIR': '/tmp/gnunet/'}, 616 haltOnFailure=True, 617 commands=[ 618 util.ShellArg(command=['./bootstrap'], logname='bootstrap', haltOnFailure=True), 619 util.ShellArg(command=['./configure', 620 "--prefix=/tmp/gnunet-buildbot", 621 "--enable-experimental", 622 "--enable-logging=verbose"], 623 logname='configure', 624 haltOnFailure=True), 625 util.ShellArg(command=['make'], 626 logname='make', 627 haltOnFailure=True), 628 util.ShellArg(command=['make', 'install'], 629 logname='make install', 630 haltOnFailure=True), 631 #util.ShellArg(command=['sudo', '/home/buildbot/bin/netjail.sh', 'bash', '-c', "'make check'"], 632 # logname='make check', 633 # warnOnFailure=True, 634 # flunkOnFailure=False), # make check has issues. 635 util.ShellArg(command=['make', 'uninstall'], 636 logname='make uninstall', 637 haltOnFailure=True)] 638 ) 639 640 gnunet_build_steps = [ 641 steps.Git(repourl='git://git.gnunet.org/gnunet.git', 642 mode='full', method='fresh'), 643 gnunet_make_step 644 ] 645 646 factory.addSteps(gnunet_build_steps) 647 648 ############################ 649 # COVERITY # 650 # Occurs: 1st day of month # 651 ############################ 652 cov_factory = util.BuildFactory() 653 cov_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='fresh')) 654 cov_factory.addStep(steps.ShellSequence( 655 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 656 env={'PATH': "${HOME}/.local/bin:${HOME}/bin:${PATH}"}, 657 commands=[ 658 util.ShellArg(command=['./bootstrap'], logname='bootstrap'), 659 util.ShellArg(command=['./configure', 660 "--prefix=/tmp/gnunet-buildbot", 661 "--enable-experimental=true"], 662 logname="configure"), 663 util.ShellArg(command=['cov-build', '--dir', 'cov-int', 'make'], logname='cov-build'), 664 ])) 665 cov_factory.addStep(steps.ShellCommand(command=['tar', 'czf', 'coverity.tar.gz', 'cov-int/'], 666 haltOnFailure=True, 667 name="Packing up")) 668 cov_factory.addStep(steps.ShellCommand(command=['curl', 669 '--form', util.Interpolate('token=%(secret:coverity_token)s'), 670 '--form', 'email=mschanzenbach@posteo.de', 671 '--form', 'version="git master"', 672 '--form', 'file=@./coverity.tar.gz', 673 '--form', 'description="Buildbot triggered build"', 674 'https://scan.coverity.com/builds?project=GNUnet%2Fgnunet'], 675 haltOnFailure=True, 676 name="Sending")) 677 678 679 ################################################# 680 # RELEASE BUILD # 681 # Occurs: # 682 # 1. Nightly # 683 # 2. Upon a pushed release tag (vX.Y.Z) # 684 # 3. With a commit message containing !tarball # 685 ################################################# 686 dist_factory = util.BuildFactory() 687 # https://github.com/buildbot/buildbot/issues/6539 change to "copy" when fixed 688 dist_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='clobber')) 689 690 dist_factory.addStep(steps.ShellSequence( 691 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 692 haltOnFailure=True, 693 commands=[ 694 util.ShellArg(command=['./bootstrap'], 695 logname='bootstrap', 696 haltOnFailure=True), 697 util.ShellArg(command=['./configure'], 698 logname='configure', 699 haltOnFailure=True), 700 util.ShellArg(command=['git', 'status'], 701 logname='status before dist', 702 haltOnFailure=True), 703 util.ShellArg(command=['make', 'dist'], 704 logname="dist", 705 haltOnFailure=True), 706 util.ShellArg(command=['make', 'doxygen'], 707 haltOnFailure=True, 708 logname="Doxygen") 709 ])) 710 711 # Get version number of tarball 712 cmdmeson = r'ls -1 build/meson-dist/gnunet-*.tar.gz | sed "s/build\/meson-dist\/gnunet-//" | sed "s/.tar.gz//" | tail -n1' 713 #cmd = 'git describe --tags | sed "s/^v//"' 714 dist_factory.addStep(steps.SetPropertyFromCommand(hideStepIf=False, 715 command=cmdmeson, 716 property='gnunet_meson_releasever', 717 name="Getting release version")) 718 dist_factory.addStep(steps.ShellCommand(command=["tar", 719 "xf", 720 util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 721 '-C', 'build'], 722 haltOnFailure=True, 723 name="Extracting tarball")) 724 725 # Make doxygen 726 # Try to build dist package 727 dist_factory.addStep(steps.ShellSequence( 728 workdir=util.Interpolate('build/build/gnunet-%(prop:gnunet_meson_releasever)s'), 729 name=util.Interpolate("GNUnet %(prop:gnunet_meson_releasever)s tarball build"), 730 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 731 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"]}, 732 commands=[ 733 util.ShellArg(command=['mkdir', '-p', '$TMPDIR'],logname='tmpdir',haltOnFailure=True), 734 util.ShellArg(command=['./configure', 735 "--prefix=/tmp/gnunet-buildbot", 736 "--enable-experimental=true", 737 "--enable-logging=verbose", 738 "--mesonbuilddir=tarball_build"], 739 logname='setup', 740 haltOnFailure=True), 741 util.ShellArg(command=['make'], 742 logname='compile', 743 haltOnFailure=True), 744 util.ShellArg(command=['make', 'install'], 745 logname='install', 746 haltOnFailure=True), 747 util.ShellArg(command=['make', 'uninstall'], 748 logname='uninstall', 749 haltOnFailure=True)] 750 )) 751 752 # Upload artifact to https://buildbot.gnunet.org/artifacts 753 dist_factory.addStep(steps.FileUpload(workersrc=util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 754 mode=0o644, 755 masterdest=util.Interpolate("~/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"), 756 url=util.Interpolate("https://buildbot.gnunet.org/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"))) 757 758 759 # Update doxygen (TODO skit on nightly?) 760 dist_factory.addStep(steps.ShellSequence( 761 name=util.Interpolate("Deploy Doxygen"), 762 workdir=util.Interpolate('build/doc'), 763 commands=[ 764 util.ShellArg(command=['chmod', '-R', 'ag+rX', '../doc'], 765 logname='Permissions', 766 haltOnFailure=True), 767 util.ShellArg(command=['rsync', '-a', '--delete', 768 '../doc/doxygen', 769 'handbook@firefly.gnunet.org:~/doc_deployment/'], 770 logname='Deploy', 771 haltOnFailure=True), 772 ])) 773 774 775 776 777 ########################### 778 # Fedora COPR build # 779 ########################### 780 copr_factory = util.BuildFactory() 781 copr_factory.addStep(steps.ShellCommand(command=["curl", 782 "-H", "Content-Type: application/json", 783 "--data", "{}", 784 "-X", "POST", 785 "https://copr.fedorainfracloud.org/webhooks/custom/36992/d316c169-1482-4508-a765-cfb5c0efeb67/gnunet/"], 786 haltOnFailure=True, 787 name="Triggering Copr build")) 788 789 790 ########################### 791 # gnunet-gns-registrar # 792 ########################### 793 registrar_factory = util.BuildFactory() 794 registrar_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-gns-registrar.git', mode='incremental')) 795 add_default_pipeline(registrar_factory) 796 797 ########################### 798 # stage.gnunet.org # 799 ########################### 800 www_factory = util.BuildFactory() 801 www_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental')) 802 add_default_pipeline(www_factory) 803 804 ########################### 805 # www.gnunet.org # 806 ########################### 807 www_stable_factory = util.BuildFactory() 808 www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental', branch='stable')) 809 add_default_pipeline(www_stable_factory) 810 811 ########################### 812 # bib.gnunet.org # 813 ########################### 814 bib_factory = util.BuildFactory() 815 bib_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunetbib.git', mode='incremental')) 816 add_default_pipeline(bib_factory) 817 818 819 ########################### 820 # reclaim.gnunet.org # 821 ########################### 822 reclaim_www_stable_factory = util.BuildFactory() 823 reclaim_www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www-reclaim.git', mode='incremental', branch='master')) 824 add_default_pipeline(reclaim_www_stable_factory) 825 826 827 ########################### 828 # rest.gnunet.org # 829 ########################### 830 rest_factory = util.BuildFactory() 831 rest_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-rest-api.git', mode='incremental', branch='master')) 832 add_default_pipeline(rest_factory) 833 834 ########################### 835 # lsd.gnunet.org # 836 ########################### 837 lsd_factory = util.BuildFactory() 838 lsd_factory.addStep(steps.Git(repourl=util.Property('repository'), mode='full', method="clobber", branch='master')) 839 add_default_pipeline(lsd_factory) 840 841 ########################### 842 # gana.gnunet.org # 843 ########################### 844 gana_factory = util.BuildFactory() 845 gana_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gana.git', mode='full', method="fresh", branch='master')) 846 add_default_pipeline(gana_factory) 847 848 ############################## 849 # doc.gnunet.org (Doc-NG) # 850 ############################## 851 doc_factory = util.BuildFactory() 852 doc_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-handbook.git', alwaysUseLatest=True, mode='full', method="clobber", branch='master')) 853 add_default_pipeline(doc_factory) 854 855 856 857 ########################### 858 # Buildbot TODO delete at some point # 859 ########################### 860 bb_factory = util.BuildFactory() 861 bb_factory.addStep(steps.Git(repourl='ssh://git@git.gnunet.org/buildbot-ci.git', mode='incremental')) 862 bb_factory.addStep(steps.ShellCommand(command=["./reload_bb-master.sh"], name="Reload configuration")) 863 864 865 866 BUILDERS.append( 867 util.BuilderConfig(name="gnunet-debian-x86_64", 868 workernames=["firefly-x86_64-amdepyc"], 869 factory=factory)) 870 871 BUILDERS.append( 872 util.BuilderConfig(name="gnunet-fedora-aarch64", 873 workernames=["schanzen-aarch64-fedora-meson"], 874 factory=factory)) 875 876 BUILDERS.append( 877 util.BuilderConfig(name="gnunet-debian-x86_64-dev", 878 workernames=["firefly-x86_64-amdepyc"], 879 factory=factory)) 880 881 BUILDERS.append( 882 util.BuilderConfig(name="gnunet-fedora-aarch64-dev", 883 workernames=["schanzen-aarch64-fedora-meson"], 884 factory=factory)) 885 886 887 BUILDERS.append( 888 util.BuilderConfig(name="gnunet_release", 889 workernames=["firefly-x86_64-amdepyc"], 890 factory=dist_factory)) 891 BUILDERS.append( 892 util.BuilderConfig(name="gnunet_coverity", 893 workernames=["firefly-x86_64-amdepyc"], 894 factory=cov_factory)) 895 896 BUILDERS.append( 897 util.BuilderConfig(name="gnunet-gns-registrar", 898 workernames=["firefly-x86_64-amdepyc", "schanzen-aarch64-fedora-meson"], 899 factory=registrar_factory)) 900 BUILDERS.append( 901 util.BuilderConfig(name="stage.gnunet.org", 902 workernames=["firefly-x86_64-amdepyc"], 903 factory=www_factory)) 904 BUILDERS.append( 905 util.BuilderConfig(name="www.gnunet.org", 906 workernames=["firefly-x86_64-amdepyc"], 907 factory=www_stable_factory)) 908 BUILDERS.append( 909 util.BuilderConfig(name="bib.gnunet.org", 910 workernames=["firefly-x86_64-amdepyc"], 911 factory=bib_factory)) 912 BUILDERS.append( 913 util.BuilderConfig(name="reclaim.gnunet.org", 914 workernames=["firefly-x86_64-amdepyc"], 915 factory=reclaim_www_stable_factory)) 916 BUILDERS.append( 917 util.BuilderConfig(name="rest.gnunet.org", 918 workernames=["firefly-x86_64-amdepyc"], 919 factory=rest_factory)) 920 BUILDERS.append( 921 util.BuilderConfig(name="lsd.gnunet.org", 922 workernames=["firefly-x86_64-amdepyc"], 923 factory=lsd_factory)) 924 BUILDERS.append( 925 util.BuilderConfig(name="gana.gnunet.org", 926 workernames=["firefly-x86_64-amdepyc"], 927 factory=gana_factory)) 928 BUILDERS.append( 929 util.BuilderConfig(name="doc.gnunet.org", 930 workernames=["firefly-x86_64-amdepyc"], 931 factory=doc_factory)) 932 BUILDERS.append( 933 util.BuilderConfig(name="buildbot", 934 workernames=["firefly-x86_64-amdepyc"], 935 factory=bb_factory)) 936 BUILDERS.append( 937 util.BuilderConfig(name="gnunet_rpm_copr", 938 workernames=["firefly-x86_64-amdepyc"], 939 factory=copr_factory)) 940 941 942 bsg = reporters.BuildStatusGenerator(mode=["exception", "failing", "problem"], 943 builders=["gnunet-debian-x86_64", "gnunet-fedora-aarch64"]) 944 945 mn = reporters.MailNotifier(fromaddr="buildbot@firefly.gnunet.org", 946 sendToInterestedUsers=False, 947 extraRecipients=["gnunet-ci@gnunet.org"], 948 lookup="gnunet.org", 949 generators=[bsg]) 950 SERVICES.append(mn) 951 952 953 954 955 ############################################# 956 # 19: CONTAINER FACTORY ##################### 957 ############################################# 958 ## 959 # These factories uses the standard container worker. 960 WORKERS.append(Worker("container-worker", "container-pass")) 961 962 963 # Container Job Generator Functions 964 # Return the standard configuration for a container job. 965 def default_job_config(jobName, repoName): 966 return {jobName: {"HALT_ON_FAILURE": True, 967 "WARN_ON_FAILURE": False, 968 "CONTAINER_BUILD": True, 969 "CONTAINER_NAME": repoName, 970 "CONTAINER_ARCH": "amd64"}} 971 972 973 # Parse a job config and apply it as overrides to the standard configuration. 974 def ingest_job_config(configText, jobName, repoName): 975 configDict = default_job_config(jobName, repoName) 976 jobIni = configparser.ConfigParser() 977 jobIni.optionxform = str 978 jobIni.read_string(configText) 979 configDict[jobName].update(jobIni["build"]) 980 print(configDict) 981 return configDict 982 983 984 # Search for configs, and ingest 985 def handle_job_config(jobName, repoName, configText, configResult): 986 print(configText) 987 if configResult == util.SUCCESS: 988 print(f"Ingesting Job Config: {configText}") 989 configDict = ingest_job_config(configText, jobName, repoName) 990 print(configDict) 991 return configDict 992 else: 993 print("No job config; Using default params") 994 return default_job_config(jobName, repoName) 995 996 997 class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): 998 999 def __init__(self, REPO_NAME, **kwargs): 1000 self.REPO_NAME = REPO_NAME 1001 kwargs = self.setupShellMixin(kwargs) 1002 super().__init__(**kwargs) 1003 self.observer = logobserver.BufferLogObserver() 1004 self.addLogObserver('stdio', self.observer) 1005 1006 def extract_stages(self, stdout): 1007 stages = [] 1008 for line in stdout.split('\n'): 1009 stage = str(line.strip()) 1010 if stage: 1011 stages.append(stage) 1012 return stages 1013 1014 @defer.inlineCallbacks 1015 def run(self): 1016 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.REPO_NAME}" 1017 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1018 # run 'ls <project_root>/contrib/ci/jobs/' to get the list of stages 1019 cmd = yield self.makeRemoteShellCommand() 1020 yield self.runCommand(cmd) 1021 jobDirs = [] 1022 1023 # if the command passes extract the list of stages 1024 result = cmd.results() 1025 if result == util.SUCCESS: 1026 jobDirs = self.extract_stages(self.observer.getStdout()) 1027 print(f"this is jobDirs list: {jobDirs}") 1028 self.configDict = {} 1029 print(f"Remote cmd stdout: {self.observer.getStdout()}") 1030 print(f"cmd.results: {cmd.results()}") 1031 for stage in jobDirs: 1032 jobDirPath = f"{CI_JOBS_PATH}/{stage}" 1033 observer = logobserver.BufferLogObserver() 1034 self.addLogObserver('stdio', observer) 1035 cmd1 = yield self.makeRemoteShellCommand( 1036 command=["cat", f"{jobDirPath}/config.ini"]) 1037 yield self.runCommand(cmd1) 1038 print(f"cmd1.results: {cmd1.results()}") 1039 print(f"Second command stdout: {observer.getStdout()}") 1040 print(f"Current stage: {stage}") 1041 print(jobDirPath) 1042 self.configDict.update( 1043 handle_job_config( 1044 stage, self.REPO_NAME, 1045 observer.getStdout(), cmd1.results())) 1046 print(self.configDict) 1047 # create a container step for each stage and 1048 # add them to the build 1049 convstr2bool = ast.literal_eval 1050 self.build.addStepsAfterCurrentStep([ 1051 container_add_step( 1052 convstr2bool( 1053 str(self.configDict[stage]["HALT_ON_FAILURE"])), 1054 convstr2bool( 1055 str(self.configDict[stage]["WARN_ON_FAILURE"])), 1056 convstr2bool( 1057 str(self.configDict[stage]["CONTAINER_BUILD"])), 1058 self.configDict[stage]["CONTAINER_NAME"], 1059 container_factory, 1060 CONTAINER_WORKDIR, 1061 self.REPO_NAME, 1062 stage, 1063 self.configDict[stage]["CONTAINER_ARCH"], 1064 f"contrib/ci/jobs/{stage}/job.sh") 1065 for stage in jobDirs 1066 ]) 1067 1068 return result 1069 1070 # MHD2 uses a deployment-owned Debian overlay and a reviewed upstream pin. 1071 # Both architectures must pass before either is uploaded to the nightly feed. 1072 MHD2_WORKDIR = "/home/container-worker/workspace/libmicrohttpd2-ci" 1073 mhd2_factory = util.BuildFactory() 1074 mhd2_factory.addStep(Git( 1075 name="git", repourl="git://git.taler.net/taler-deployment.git", 1076 branch="master", mode="full", method="fresh", workdir=MHD2_WORKDIR, 1077 haltOnFailure=True)) 1078 for arch in ("amd64", "arm64"): 1079 mhd2_factory.addStep(container_add_step( 1080 True, False, True, "unused", mhd2_factory, MHD2_WORKDIR, 1081 "libmicrohttpd2", f"build-{arch}", arch, 1082 "buildbot/libmicrohttpd2/build.sh", 1083 "buildbot/libmicrohttpd2/Containerfile")) 1084 mhd2_factory.addStep(container_add_step( 1085 True, False, False, "localhost/buildbot/libmicrohttpd2/build-amd64:amd64", 1086 mhd2_factory, MHD2_WORKDIR, "libmicrohttpd2", "upload", "amd64", 1087 "buildbot/libmicrohttpd2/upload.sh")) 1088 BUILDERS.append(util.BuilderConfig( 1089 name="libmicrohttpd2-builder", workernames=["container-worker"], 1090 factory=mhd2_factory)) 1091 1092 1093 def mhd2_packaging_changed(change): 1094 return any(path == "buildbot/master.cfg" or path.startswith(( 1095 "buildbot/libmicrohttpd2/", 1096 "packaging/ng/buildscripts/debian-overlays/libmicrohttpd2/")) 1097 for path in change.files) 1098 1099 1100 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1101 name="libmicrohttpd2-packaging-scheduler", 1102 change_filter=util.ChangeFilter(branch="master", project_re="^taler-deployment$"), 1103 fileIsImportant=mhd2_packaging_changed, treeStableTimer=30, 1104 builderNames=["libmicrohttpd2-builder"])) 1105 1106 # List of repos to add to container factory. 1107 container_repos = ["git.gnunet.org/gnunet", 1108 "git.taler.net/anastasis", 1109 "git.taler.net/challenger", 1110 "git.taler.net/donau", 1111 "git.taler.net/exchange", 1112 "git.taler.net/libeufin", 1113 "git.taler.net/taler-rust", 1114 "git.taler.net/depolymerization", 1115 "git.taler.net/merchant", 1116 "git.taler.net/sync", 1117 "git.taler.net/taler-android", 1118 "git.taler.net/taler-mailbox", 1119 "git.taler.net/taldir", 1120 "git.taler.net/taler-typescript-core",] 1121 1122 for repo in container_repos: 1123 1124 # Factory-wide variables 1125 REPO_NAME = repo.rsplit('/', 1)[1] 1126 REPO_URL = "git://" + repo + ".git" 1127 CACHE_CHECKOUT = REPO_NAME in ( 1128 "gnunet", "exchange", "merchant", "donau", "challenger", "anastasis", 1129 "sync", "libeufin", "taler-rust", "depolymerization", "taler-mailbox", 1130 "taldir", "taler-typescript-core", 1131 ) 1132 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{REPO_NAME}" 1133 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1134 1135 # Create a factory 1136 container_factory = util.BuildFactory() 1137 container_factory.workdir = CONTAINER_WORKDIR 1138 1139 # Containers can leave files owned by root. Make them writable before cleanup. 1140 workspace_command = ( 1141 f"if test -d {CONTAINER_WORKDIR}; then " 1142 f"podman run --log-driver=none --rm --volume {CONTAINER_WORKDIR}:/workdir " 1143 "docker.io/library/debian:bookworm-slim chmod -R 777 /workdir; fi" 1144 ) 1145 if CACHE_CHECKOUT: 1146 # Replace the previous shallow clone once, then retain the full checkout. 1147 workspace_command += ( 1148 f" && if test -f {CONTAINER_WORKDIR}/.git/shallow; then " 1149 f"rm -rf {CONTAINER_WORKDIR}; fi" 1150 ) 1151 else: 1152 workspace_command += f" && rm -rf {CONTAINER_WORKDIR}" 1153 workspace_command += f" && mkdir -p {CONTAINER_WORKDIR}" 1154 1155 container_factory.addStep(ShellCommand( 1156 name="workspace", 1157 descriptionDone="Workspace directory check", 1158 command=workspace_command, 1159 haltOnFailure=True, 1160 )) 1161 1162 # "fresh" cleans build outputs but retains Git history between builds. 1163 # Debian package versions require full history and all release tags. 1164 # Other container builders use a new depth-1 clone on every build. 1165 # Will checkout value of "branch" property from job properties. 1166 # https://docs.buildbot.net/latest/manual/configuration/steps/source_git.html 1167 container_factory.addStep(Git( 1168 name="git", 1169 repourl=REPO_URL, 1170 branch=util.Interpolate('%(src::branch)s'), 1171 mode='full', 1172 method='fresh' if CACHE_CHECKOUT else 'clobber', 1173 shallow=not CACHE_CHECKOUT, 1174 tags=CACHE_CHECKOUT, 1175 submodules=True, 1176 haltOnFailure=True, 1177 )) 1178 1179 container_factory.addStep(GenerateStagesCommand( 1180 REPO_NAME, 1181 name="Generate build stages", 1182 command=f"ls {CI_JOBS_PATH}", 1183 haltOnFailure=True)) 1184 1185 BUILDERS.append(util.BuilderConfig( 1186 name=f"{REPO_NAME}-builder", 1187 workernames=["container-worker"], 1188 factory=container_factory 1189 )) 1190 1191 # Only enable this scheduler for debugging! 1192 # Will run builders with 1 minute of waiting inbetween builds 1193 # SCHEDULERS.append(schedulers.Periodic( 1194 # name=f"{REPO_NAME}-minutely", 1195 # builderNames=[f"{REPO_NAME}-builder"], 1196 # periodicBuildTimer=60 1197 # )) 1198 1199 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1200 name=f"{REPO_NAME}-container-scheduler", 1201 change_filter=util.ChangeFilter( 1202 branch="master", 1203 # Anchored: ChangeFilter matches the regex against the start of 1204 # the project name only, so an unanchored "(anastasis)" would 1205 # also fire for pushes to 'anastasis-www' and 'anastasis-gtk'. 1206 project_re=f"^({REPO_NAME})$" 1207 ), 1208 treeStableTimer=30, 1209 builderNames=[f"{REPO_NAME}-builder"] 1210 )) 1211 1212 SERVICES.append(reporters.MailNotifier( 1213 fromaddr="buildbot@taler.net", 1214 # notify from pass to fail, and viceversa. 1215 generators=[BuildStatusGenerator( 1216 mode=('change','problem','failing','exception',), 1217 builders=[f"{REPO_NAME}-builder",], 1218 message_formatter=reporters.MessageFormatter( 1219 template_type='plain', 1220 want_logs_content=True, 1221 ), 1222 )], 1223 sendToInterestedUsers=False, 1224 useTls=False, 1225 relayhost="localhost", 1226 smtpPort=25, 1227 dumpMailsToLog=True, 1228 extraRecipients=[f"ci-{REPO_NAME}@taler.net"] 1229 )) 1230 1231 1232 ################ 99: debug stuff JOB ################################### 1233 1234 # This does nothing, just a starting point for a factory. 1235 DEBUG_FACTORY = util.BuildFactory() 1236 DEBUG_FACTORY.addStep( 1237 ShellCommand( 1238 name="echo debug", 1239 description="just echoing a word", 1240 descriptionDone="builder responded", 1241 command=["echo", "I'm here!"] 1242 ) 1243 ) 1244 1245 1246 ################################################################## 1247 #################### General purpose ############################# 1248 ################################################################## 1249 1250 # Compute array of the names of all of our builders 1251 BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) 1252 1253 ####### GENERAL PURPOSE BUILDBOT SERVICES ####################### 1254 1255 SERVICES.append(reporters.MailNotifier( 1256 fromaddr="testbuild@taler.net", 1257 # notify from pass to fail, and viceversa. 1258 generators=[BuildStatusGenerator( 1259 mode=('change','problem','failing','exception',), 1260 builders=EMAIL_ALERTS, 1261 message_formatter=reporters.MessageFormatter( 1262 template_type='plain', 1263 want_logs_content=True, 1264 ), 1265 )], 1266 sendToInterestedUsers=False, 1267 useTls=False, 1268 relayhost="localhost", 1269 smtpPort=25, 1270 dumpMailsToLog=True, 1271 extraRecipients=BUILDER_EMAIL_ADDRESSES 1272 )) 1273 1274 1275 ############# GENERAL PURPOSE SCHEDULERS ########################## 1276 1277 # Workers that are done on wallet or deployment changes to master 1278 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1279 name="taler-healthcheck-scheduler", 1280 change_filter=util.ChangeFilter( 1281 branch="master", 1282 project_re="(taler-typescript-core|taler-deployment)" 1283 ), 1284 treeStableTimer=None, 1285 builderNames=WALLETCHANGE_TRIGGERS 1286 )) 1287 1288 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1289 name="all-scheduler", 1290 change_filter=util.ChangeFilter( 1291 branch_re="(master|stable)", 1292 project_re="(taler-typescript-core|exchange|" 1293 "merchant|taler-deployment|twister|sync|" 1294 "taler-merchant-demos)" 1295 ), 1296 treeStableTimer=None, 1297 builderNames=CODECHANGE_TRIGGERS 1298 )) 1299 1300 # Scheduler for all nightly builds. 1301 SCHEDULERS.append(schedulers.Nightly( 1302 name="nightly-scheduler", 1303 builderNames=list(NIGHTLY_TRIGGERS), 1304 branch="master", 1305 hour=6, 1306 minute=0 1307 )) 1308 1309 # Provide "force" button in the web UI. 1310 SCHEDULERS.append(schedulers.ForceScheduler( 1311 name="force-scheduler", 1312 buttonName="Force build", 1313 builderNames=list(BUILDER_LIST) 1314 )) 1315 1316 ######################################################### 1317 ####### Actual configuation initialization ############## 1318 ######################################################### 1319 1320 # This is the dictionary that the buildmaster pays attention to. We also use 1321 # a shorter alias to save typing. 1322 c = BuildmasterConfig = {} 1323 1324 # Secrets 1325 c['secretsProviders'] = [secrets.SecretInAFile(dirname="/home/buildbot-master/secrets")] 1326 1327 c["workers"] = WORKERS 1328 c["builders"] = BUILDERS 1329 c["schedulers"] = SCHEDULERS 1330 c["services"] = SERVICES 1331 1332 # Silence warning and allow very basic phoning home. 1333 c["buildbotNetUsageData"] = "basic" 1334 1335 c["title"] = "GNUnet Builder" 1336 c["titleURL"] = "https://buildbot.gnunet.org" 1337 1338 # the 'buildbotURL' string should point to the location where the buildbot's 1339 # internal web server is visible. 1340 c['buildbotURL'] = "https://buildbot.gnunet.org/" 1341 1342 # This specifies what database buildbot uses to store its 1343 # state. You can leave this at its default for all but the 1344 # largest installations. 1345 c["db"] = { 1346 "db_url": "sqlite:///state.sqlite", 1347 } 1348 1349 # the 'change_source' setting tells the buildmaster how it should 1350 # find out about source code changes. 1351 pbSource = changes.PBChangeSource(port="tcp:19990:interface=127.0.0.1", 1352 user="git-buildbot", 1353 passwd="Aer3eari") 1354 1355 pollGnunetSource = changes.GitPoller(repourl='git://git.gnunet.org/gnunet.git', 1356 branches=True, 1357 pollInterval=300, 1358 pollAtLaunch=True, 1359 project="gnunet") 1360 c["change_source"] = [pollGnunetSource, pbSource] 1361 1362 # 'protocols' contains information about protocols which master 1363 # will use for communicating with workers. You must define at 1364 # least 'port' option that workers could connect to your master 1365 # with this protocol. 'port' must match the value configured into 1366 # the workers (with their --master option) 1367 c["protocols"] = {"pb": {"port": "tcp:19989"}} 1368 1369 # Load admin password 1370 with open("/home/buildbot-master/secrets/admin") as fh: 1371 www_pass = fh.read().strip() 1372 1373 # minimalistic config to activate new web UI 1374 # -- formerly commented out as not packaged properly in Debian and others, see 1375 # https://bugzilla.redhat.com/show_bug.cgi?id=1557687 1376 c["www"] = { 1377 "port": "tcp:8009:interface=127.0.0.1", 1378 "default_page": 'builders', 1379 "plugins": { 1380 "waterfall_view": True, 1381 "console_view": True, 1382 "grid_view": True, 1383 }, 1384 "auth": util.UserPasswordAuth([('admin',www_pass)]), 1385 "allowed_origins": ["https://*.taler.net","https://*.gnunet.org"], 1386 "avatar_methods": [], 1387 }