master.cfg (58689B)
1 # -*- python -*- 2 # ex: set syntax=python: 3 4 ## 5 # This file is part of TALER 6 # (C) 2016-2025 Taler Systems SA 7 # 8 # TALER is free software; you can redistribute it and/or 9 # modify it under the terms of the GNU Affero General Public 10 # License as published by the Free Software Foundation; either 11 # version 3, or (at your option) any later version. 12 # 13 # TALER is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty 15 # of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 16 # See the GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public 19 # License along with TALER; see the file COPYING. If not, 20 # see <http://www.gnu.org/licenses/> 21 # 22 # @author Florian Dold 23 # @author Marcello Stanisci 24 # @author ng0 25 # @author Christian Grothoff 26 # @author Devan Carpenter 27 import ast 28 import configparser 29 import glob 30 import os 31 import pathlib 32 import pwd 33 import re 34 import subprocess 35 36 from buildbot.changes.pb import PBChangeSource 37 from buildbot.steps.source.git import Git 38 from buildbot.steps.shell import ShellCommand 39 from buildbot.plugins import changes 40 from buildbot.plugins import reporters 41 from buildbot.plugins import schedulers 42 from buildbot.plugins import steps 43 from buildbot.plugins import secrets, util 44 from buildbot.process import buildstep, logobserver 45 from buildbot.process.results import SKIPPED 46 from buildbot.reporters.generators.build import BuildStatusGenerator 47 from buildbot.worker import Worker 48 from twisted.internet import defer 49 50 # This is a sample buildmaster config file. It must be 51 # installed as 'master.cfg' in your buildmaster's base 52 # directory. 53 54 # This file has the following structure: 55 # - Globals: definition of global variables we use throughout 56 # + Convenience functions: helper functions useful for many jobs 57 # - Jobs: actual job definitions 58 # - General purpose: triggers and alerts shared by various jobs 59 # + general purpose notification (alerts) 60 # + general purpose triggers (schedulers) 61 # - Actual buildbot configuration object initialization 62 63 ################################################################# 64 ######################### GLOBALS ############################### 65 ################################################################# 66 67 # The 'workers' list defines the set of recognized workers. 68 # Each element is a Worker object, specifying a unique worker 69 # name and password. The same worker name and password must 70 # be configured on the worker. 71 WORKERS = [] 72 73 # 'services' is a list of BuildbotService items like reporter 74 # targets. The status of each build will be pushed to these 75 # targets. buildbot/reporters/*.py has a variety to choose from, 76 # like IRC bots. 77 78 79 class MessageFormatterWithStdout(reporters.MessageFormatter): 80 def buildAdditionalContext(self, master, ctx): 81 stdout = [] 82 for step in ctx["build"]["steps"]: 83 for log in step["logs"]: 84 all_logs = log["content"]["content"].splitlines() 85 # Including only what the script printed on stdout. 86 for line in all_logs: 87 if re.search("^o", line): 88 stdout.append(line[1:]) 89 ctx.update(dict(stdout="\n".join(stdout))) 90 91 92 ##################################################### 93 # Commit message triggers # 94 # # 95 # This checks for triggers in the commit messages # 96 # !tarball will trigger a release build # 97 # !coverity will trigger a coverity check # 98 ##################################################### 99 def checkForTarballTrigger(change): 100 if "!tarball" in change.comments: 101 return True 102 return False 103 def checkForCoverityTrigger(change): 104 if "!coverity" in change.comments: 105 return True 106 107 SERVICES = [] 108 109 # The 'builders' list defines the Builders, which tell Buildbot 110 # how to perform a build: what steps, and which workers can execute 111 # them. Note that any particular build will only take place on 112 # one worker. 113 BUILDERS = [] 114 115 # Configures the Schedulers, which decide how to react to incoming 116 # changes. 117 SCHEDULERS = [] 118 119 # Array of builders to be scheduled every night. 120 NIGHTLY_TRIGGERS=[] 121 122 # Array of builders to be scheduled whenever any of the code Git repos change 123 CODECHANGE_TRIGGERS = [] 124 125 # Array of builders to be scheduled whenever the taler-typescript-core or 126 # taler-deployment change 127 WALLETCHANGE_TRIGGERS = [] 128 129 # Array of builder names for which build status reports should be sent 130 # via e-mail 131 EMAIL_ALERTS = [] 132 133 # Array of email address for which build status reports shoudl be sent 134 BUILDER_EMAIL_ADDRESSES = [] 135 136 ############ Convenience functions ################# 137 138 # Create a FACTORY with a taler-deployment.git checkout as the first step. 139 def create_factory_with_deployment(): 140 f = util.BuildFactory() 141 update_deployment(f) 142 return f 143 144 # Convenience function that checks out a Git repository. 145 # First argument is the URL of the Git to clone, second 146 # the desired branch. Default is 'master'. 147 def git_step(repo, target_branch="master"): 148 return Git( 149 repourl=repo, 150 mode="full", 151 method="fresh", 152 logEnviron=False, 153 alwaysUseLatest=True, 154 haltOnFailure=True, 155 branch=target_branch 156 ) 157 158 # FIXME: document this function! 159 def add_default_step(bldr, step): 160 worker_script = ".buildbot/%(prop:workername)s_" + step + ".sh" 161 default_script = ".buildbot/" + step + ".sh" 162 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (worker_script,worker_script) 163 bldr.addStep(steps.SetPropertyFromCommand(command=util.Interpolate(cmd), 164 hideStepIf=True, 165 property='buildbotScript_%s' % step, 166 name="Checking for worker-specific %s script" % step)) 167 cmd = '[ -f %s ] && ls -1 %s || exit 0' % (default_script,default_script) 168 bldr.addStep(steps.SetPropertyFromCommand(doStepIf=(util.Property('buildbotScript_%s' % step) != util.Interpolate(worker_script)), 169 hideStepIf=True, 170 command=util.Interpolate(cmd), 171 property='buildbotScript_%s' % step, 172 name="Checking for %s script" % step)) 173 bldr.addStep(steps.ShellCommand(command=util.Property('buildbotScript_%s' % step), 174 haltOnFailure=True, 175 env={'PATH': "${HOME}/.local/bin:$HOME/bin:${PATH}"}, 176 doStepIf=(util.Property('buildbotScript_%s' % step) != None), 177 hideStepIf=lambda results, s: results==SKIPPED, 178 name="Executing %s step" % step)) 179 180 181 182 # Convenience function that runs 'make check' in a 183 # directory of the code inside of a netjail. 184 def jailed_check(package, srcdirs): 185 return steps.ShellSequence( 186 name="Tests of " + package, 187 description="Testing " + package, 188 descriptionDone="Pass", 189 commands=map(lambda srcdir: util.ShellArg(command=["sudo", "/usr/local/bin/netjail.sh", "/home/container-worker/taler-deployment/buildbot/with-postgres.sh", "bash", "-c", "'cd src/"+srcdir+" make install check'"]), srcdirs), 190 env={'PATH': "${HOME}/local/bin:${PATH}"}, 191 workdir="../../sources/" + package 192 ) 193 194 # Convenience function that checks out the deployment. 195 def update_deployment(factory): 196 factory.addStep(steps.ShellSequence( 197 name="update taler-deployment", 198 description="removing old deployment and fetching fresh repository", 199 descriptionDone="Deployment updated", 200 commands=[ 201 util.ShellArg(command=["rm", "-rf", "taler-deployment"]), 202 util.ShellArg(command=["git", "clone", "git://git.taler.net/taler-deployment"]), 203 ], 204 haltOnFailure=True, 205 workdir="../.." 206 )) 207 208 # Convenience function that builds and runs a container. 209 def container_add_step(HALT_ON_FAILURE, 210 WARN_ON_FAILURE, 211 CONTAINER_BUILD, 212 CONTAINER_NAME, 213 factory, 214 WORK_DIR, 215 repoName, 216 stepName, 217 CONTAINER_ARCH="amd64", 218 jobCmd="/workdir/contrib/ci/ci.sh", 219 containerFile="contrib/ci/Containerfile", 220 RUN_ON="push"): 221 print(f"HALT_ON_FAILURE: {HALT_ON_FAILURE}, WARN_ON_FAILURE: {WARN_ON_FAILURE}, CONTAINER_BUILD: {CONTAINER_BUILD}, CONTAINER_NAME: {CONTAINER_NAME}") 222 223 # Locally built images belong to this repository, job and architecture. 224 # A copied CONTAINER_NAME must not overwrite another builder's image 225 # between its build and run commands on the shared container worker. 226 # CONTAINER_NAME selects the image only for jobs using prebuilt images. 227 imageName = (f"localhost/buildbot/{repoName}/{stepName}:{CONTAINER_ARCH}" 228 if CONTAINER_BUILD else CONTAINER_NAME) 229 230 runCommand = ["podman", "--transient-store", "run", "--rm", 231 "--arch", CONTAINER_ARCH, 232 "--log-driver=none", 233 "--add-host", "taler.host.internal:10.0.2.2", 234 "--network", "slirp4netns:allow_host_loopback=true", 235 "--env", util.Interpolate("CI_COMMIT_REF=%(prop:got_revision:-%(src::revision:-unknown)s)s"), 236 "--env", util.Interpolate("CI_GIT_BRANCH=%(src::branch)s"), 237 "--env", util.Interpolate("CI_PROJECT_NAME=%(src::project)s"), 238 "--cap-add", "SYS_ADMIN,CAP_SYS_CHROOT", 239 "--volume", f"{WORK_DIR}:/workdir", 240 "--volume", "/home/container-worker/ephemeral_ci_artifacts:/artifacts", 241 "--volume", "/home/container-worker/persistent_ci_artifacts:/persistent_artifacts", 242 "--volume", "/home/container-worker/mounted_files/ci_container_id_ed25519:/root/.ssh/id_ed25519:ro", 243 "--volume", "/home/container-worker/mounted_files/container_known_hosts:/root/.ssh/known_hosts:ro", 244 "--workdir", "/workdir"] 245 246 # Inputs directory 247 inputs_path = f"/home/container-worker/container_inputs/{repoName}" 248 print(f"Checking that {inputs_path} exists") 249 if os.path.isdir(inputs_path): 250 print(f"Adding {inputs_path}") 251 runCommand += ["--volume", f"{inputs_path}:/inputs:ro"] 252 else: 253 print(f"Inputs directory not found at {inputs_path}") 254 255 if CONTAINER_BUILD: 256 runCommand += ["--volume", f"/run/user/{pwd.getpwnam('container-worker').pw_uid}/podman/podman.sock:/run/podman/podman.sock", 257 "--security-opt", "label=disable"] 258 259 runCommand += [imageName, jobCmd] 260 261 # Halting jobs (including deployment) remain fatal even if they also 262 # request warnings. Only non-halting jobs may be advisory. 263 advisory = WARN_ON_FAILURE and not HALT_ON_FAILURE 264 runArg = util.ShellArg(command=runCommand, 265 logname='run inside container', 266 flunkOnFailure=not advisory, 267 warnOnFailure=advisory, 268 haltOnFailure=HALT_ON_FAILURE) 269 270 buildArg = util.ShellArg(command=["podman", "build", "-t", imageName, 271 "--arch", CONTAINER_ARCH, 272 "-f", containerFile, "."], 273 logname='build container', haltOnFailure=True) 274 275 # Stages come from `ls contrib/ci/jobs`, which cannot express "not in this 276 # kind of build". So each job says which schedulers it is for and the 277 # stage skips itself in the others. Without this a nightly would re-run 278 # the deployment jobs and publish a second set of nightly APKs, and a push 279 # would wait for a job meant to run once a day. 280 # 281 # `scheduler` is set by buildbot on every build a scheduler triggers, so 282 # the two names below are the two automatic cases and everything else is 283 # somebody asking for a build by hand. 284 def runs_in_this_build(step): 285 scheduler = step.getProperty("scheduler", "") or "" 286 if scheduler == f"{repoName}-nightly": 287 return RUN_ON in ("nightly", "both") 288 if scheduler == f"{repoName}-container-scheduler": 289 return RUN_ON in ("push", "both") 290 # Anything else -- a forced build, or a repository wired up by hand 291 # rather than through the container loop -- runs every stage. It was 292 # asked for explicitly, and a nightly-only job that could not be 293 # forced would be a job nobody could try out before tomorrow. 294 return True 295 296 skipped = lambda results, s: results == SKIPPED 297 298 if not CONTAINER_BUILD: 299 return steps.ShellSequence( 300 name=stepName, 301 commands=[runArg], 302 haltOnFailure=HALT_ON_FAILURE, 303 flunkOnFailure=True, 304 warnOnWarnings=True, 305 doStepIf=runs_in_this_build, 306 hideStepIf=skipped, 307 workdir=WORK_DIR 308 ) 309 else: 310 return steps.ShellSequence( 311 name=stepName, 312 commands=[buildArg, runArg], 313 haltOnFailure=HALT_ON_FAILURE, 314 flunkOnFailure=True, 315 warnOnWarnings=True, 316 doStepIf=runs_in_this_build, 317 hideStepIf=skipped, 318 workdir=WORK_DIR 319 ) 320 321 ################################################################## 322 ######################## JOBS #################################### 323 ################################################################## 324 325 # For every job, we have (in this order!): 326 # - worker(s): hosts/users that run the job 327 # - factory: list of steps that define what to do 328 # - builder: gives the job a name and binds it to the factory and worker 329 # - (OPTIONAL) alerts: notifications to trigger when the job fails 330 # Pre-defined: EMAIL_ALERTS 331 # - scheduler: rules that define when to run the job 332 # Pre-defined: NIGHTLY_TRIGGERS, CODECHANGE_TRIGGERS, WALLETCHANGE_TRIGGERS 333 334 ################ 1: BUILDMASTER JOB ################################### 335 336 ## 337 # This worker restarts the buildmaster itself on 338 # changes to this file. 339 # Location: /home/buildbot-master 340 WORKERS.append(Worker("buildmaster-worker", "buildmaster-pass")) 341 342 BUILDMASTER_FACTORY = create_factory_with_deployment() 343 BUILDMASTER_FACTORY.addStep( 344 ShellCommand( 345 name="restart buildmaster", 346 description="trigger buildmaster restart with new configuration", 347 descriptionDone="Buildmaster updated", 348 command=["pkill", "-HUP", "-A", "-u", "buildbot-master", "-f", "/usr/bin/python3"], 349 workdir="../.." 350 ) 351 ) 352 353 BUILDERS.append(util.BuilderConfig( 354 name="buildmaster-builder", 355 workernames=["buildmaster-worker"], 356 factory=BUILDMASTER_FACTORY 357 )) 358 359 EMAIL_ALERTS.append("buildmaster-builder") 360 361 # Buildmaster is notified whenever deployment.git changes 362 SCHEDULERS.append(schedulers.SingleBranchScheduler( 363 name="buildmaster-scheduler", 364 change_filter=util.ChangeFilter( 365 branch="master", 366 project_re="(taler-deployment)" 367 ), 368 treeStableTimer=None, 369 builderNames=["buildmaster-builder"] 370 )) 371 372 ################ 2: WEBSITE JOB ################################### 373 374 ## 375 # This worker builds Websites: www and stage. 376 # 377 WORKERS.append(Worker("sites-worker", "sites-pass")) 378 379 SITES_FACTORY = create_factory_with_deployment() 380 SITES_FACTORY.addStep( 381 ShellCommand( 382 name="build Web sites", 383 description="Building all the Taler homepages", 384 descriptionDone="Sites built.", 385 command=["./build-sites.sh"], 386 workdir="../../taler-deployment/worker-sites", 387 haltOnFailure=True 388 ) 389 ) 390 391 BUILDERS.append(util.BuilderConfig( 392 name="sites-builder", workernames=["sites-worker"], factory=SITES_FACTORY 393 )) 394 395 #EMAIL_ALERTS.append("sites-builder") 396 397 398 # The web page changed if 'taler-www', 'taler-tutorials', 399 # 'taler-docs' or 'twister' changed 400 def web_page(change): 401 _change = change.asDict() 402 repo = _change.get("project") 403 if repo in ["taler-docs", "taler-tutorials", "taler-www", "twister", "taler-deployment", "buywith", "taler-ops-www", "taler-systems-www", "anastasis-www", "pepsi"]: 404 return True 405 return False 406 407 408 # Sites are re-build whenever taler-deployment, taler-www, or twister changes. 409 SCHEDULERS.append(schedulers.SingleBranchScheduler( 410 name="sites-scheduler", 411 builderNames=["sites-builder"], 412 change_filter=util.ChangeFilter( 413 branch_re="(master|stable)", 414 filter_fn=web_page 415 ), 416 treeStableTimer=None 417 )) 418 419 420 ################ 3: 'check links' JOB ################################### 421 422 ## 423 # linkchecker worker checks for dead links in the Website 424 # Location: /home/linkchecker-worker 425 WORKERS.append(Worker("linkchecker-worker", "linkchecker-pass")) 426 427 # linkchecker FACTORY 428 LINKCHECKER_FACTORY = create_factory_with_deployment() 429 LINKCHECKER_FACTORY.addStep( 430 ShellCommand( 431 name="linkchecker", 432 description="Check taler.net website for broken links && Notify", 433 descriptionDone="Results of wget in buildbot logs.", 434 command=["/home/linkchecker-worker/taler-deployment/worker-linkchecker/linkchecker.sh"], 435 workdir="/home/linkchecker-worker", 436 haltOnFailure=True, 437 timeout=7200 # 2 hours 438 ) 439 ) 440 441 # linkchecker BUILDER 442 # worker at linkchecker@taler.net 443 BUILDERS.append(util.BuilderConfig( 444 name="linkchecker-builder", 445 workernames="linkchecker-worker", 446 factory=LINKCHECKER_FACTORY 447 )) 448 449 docs_generator = BuildStatusGenerator( 450 mode=('change', 'problem', 'failing', 'exception',), 451 builders=[ 452 'linkchecker-builder', 453 ], 454 message_formatter=reporters.MessageFormatter( 455 template_type='plain', 456 want_logs_content=True 457 ) 458 ) 459 460 461 SERVICES.append(reporters.MailNotifier( 462 fromaddr="bb@taler.net", 463 generators=[docs_generator], 464 sendToInterestedUsers=False, 465 useTls=False, 466 relayhost="localhost", 467 smtpPort=25, 468 dumpMailsToLog=True, 469 extraRecipients=['linkcheck@taler.net'] 470 )) 471 472 # SERVICES.append(tipReserveEmails) 473 474 NIGHTLY_TRIGGERS.append("linkchecker-builder") 475 476 #################### 477 ## GNUnet workers ## 478 #################### 479 480 WORKERS.append(Worker("firefly-x86_64-amdepyc", "pass")) 481 WORKERS.append(Worker("mp-amd64-openbsd", "Tai7zeic")) 482 WORKERS.append(Worker("schanzen-aarch64-fedora-meson", "eWi9keet")) 483 484 485 SCHEDULERS.append(schedulers.AnyBranchScheduler( 486 name="gnunet", 487 change_filter=util.ChangeFilter(branch_re='master', 488 repository='git://git.gnunet.org/gnunet.git'), 489 treeStableTimer=None, 490 builderNames=["gnunet-debian-x86_64", 491 "gnunet-fedora-aarch64"])) 492 493 SCHEDULERS.append(schedulers.AnyBranchScheduler( 494 name="gnunet-dev", 495 change_filter=util.ChangeFilter(branch_re='dev/.+', 496 repository='git://git.gnunet.org/gnunet.git'), 497 treeStableTimer=None, 498 builderNames=["gnunet-debian-x86_64-dev", 499 "gnunet-fedora-aarch64-dev"])) 500 501 SCHEDULERS.append(schedulers.SingleBranchScheduler( 502 name="tagged_release", 503 change_filter=util.ChangeFilter(branch_re='.*v[0-9]*[.][0-9]*[.][0-9]*$', 504 repository='git://git.gnunet.org/gnunet.git'), 505 treeStableTimer=None, 506 builderNames=["gnunet_release"])) 507 508 SCHEDULERS.append(schedulers.SingleBranchScheduler( 509 name="tarball", 510 onlyImportant=True, 511 change_filter=util.ChangeFilter(branch='master', 512 repository='git://git.gnunet.org/gnunet.git'), 513 fileIsImportant=checkForTarballTrigger, 514 treeStableTimer=None, 515 builderNames=["gnunet_release"])) 516 517 518 # Build a tarball nightly 519 SCHEDULERS.append(schedulers.Nightly(name='nightly', 520 change_filter=util.ChangeFilter(branch='master', 521 repository='git://git.gnunet.org/gnunet.git'), 522 builderNames=['gnunet_release'], 523 onlyIfChanged=True, 524 hour=6, minute=0)) 525 526 SCHEDULERS.append(schedulers.SingleBranchScheduler( 527 name="gnunet_cov", 528 onlyImportant=True, 529 change_filter=util.ChangeFilter(branch='master', 530 repository='git://git.gnunet.org/gnunet.git'), 531 fileIsImportant=checkForCoverityTrigger, 532 treeStableTimer=None, 533 builderNames=["gnunet_coverity"])) 534 535 SCHEDULERS.append(schedulers.SingleBranchScheduler( 536 name="gnunet_rpm", 537 change_filter=util.ChangeFilter(branch='dev/schanzen/copr', 538 repository='git://git.gnunet.org/gnunet-rpm.git'), 539 treeStableTimer=None, 540 builderNames=["gnunet_rpm_copr"])) 541 SCHEDULERS.append(schedulers.SingleBranchScheduler( 542 name="registrar_scheduler", 543 change_filter=util.ChangeFilter(branch='master', 544 repository='git://git.gnunet.org/gnunet-gns-registrar.git'), 545 treeStableTimer=None, 546 builderNames=["gnunet-gns-registrar"])) 547 548 549 # Schedule a coverity pass monthly 550 SCHEDULERS.append(schedulers.Nightly(name='nightly_cov', 551 change_filter=util.ChangeFilter(branch='master', 552 repository='git://git.gnunet.org/gnunet.git'), 553 builderNames=['gnunet_coverity'], 554 onlyIfChanged=True, 555 dayOfMonth=0, hour=3, minute=0)) 556 557 558 559 560 # 561 # WEBSITES 562 # 563 SCHEDULERS.append(schedulers.SingleBranchScheduler( 564 name="www_master_scheduler", 565 change_filter=util.ChangeFilter(branch='master', 566 repository='git://git.gnunet.org/www.git'), 567 treeStableTimer=None, 568 builderNames=["stage.gnunet.org"])) 569 570 SCHEDULERS.append(schedulers.SingleBranchScheduler( 571 name="www_stable_scheduler", 572 change_filter=util.ChangeFilter(branch='stable', 573 repository='git://git.gnunet.org/www.git'), 574 treeStableTimer=None, 575 builderNames=["www.gnunet.org"])) 576 SCHEDULERS.append(schedulers.SingleBranchScheduler( 577 name="bib_scheduler", 578 change_filter=util.ChangeFilter(branch='master', 579 repository='git://git.gnunet.org/gnunetbib.git'), 580 treeStableTimer=None, 581 builderNames=["bib.gnunet.org"])) 582 SCHEDULERS.append(schedulers.SingleBranchScheduler( 583 name="reclaim_www_scheduler", 584 change_filter=util.ChangeFilter(branch='master', 585 repository='git://git.gnunet.org/www-reclaim.git'), 586 treeStableTimer=None, 587 builderNames=["reclaim.gnunet.org"])) 588 SCHEDULERS.append(schedulers.SingleBranchScheduler( 589 name="rest_scheduler", 590 change_filter=util.ChangeFilter(branch='master', 591 repository='git://git.gnunet.org/gnunet-rest-api.git'), 592 treeStableTimer=None, 593 builderNames=["rest.gnunet.org"])) 594 SCHEDULERS.append(schedulers.SingleBranchScheduler( 595 name="lsd_scheduler", 596 change_filter=util.ChangeFilter(branch='master', 597 repository_re='git://git.gnunet.org/lsd.*'), 598 treeStableTimer=None, 599 builderNames=["lsd.gnunet.org"])) 600 SCHEDULERS.append(schedulers.SingleBranchScheduler( 601 name="gana_scheduler", 602 change_filter=util.ChangeFilter(branch='master', 603 repository='git://git.gnunet.org/gana.git'), 604 treeStableTimer=None, 605 builderNames=["gana.gnunet.org"])) 606 SCHEDULERS.append(schedulers.SingleBranchScheduler( 607 name="doc_scheduler", 608 change_filter=util.ChangeFilter(branch='master', 609 repository='git://git.gnunet.org/gnunet-handbook.git'), 610 treeStableTimer=None, 611 builderNames=["doc.gnunet.org"])) 612 613 614 def add_default_pipeline(bldr): 615 add_default_step(bldr, "build") 616 add_default_step(bldr, "install") 617 add_default_step(bldr, "test") 618 add_default_step(bldr, "deploy") 619 620 621 factory = util.BuildFactory() 622 623 624 ########################### 625 # GNUnet # 626 ########################### 627 gnunet_make_step = steps.ShellSequence( 628 name=util.Interpolate("GNUnet build"), 629 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 630 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"], 631 'TMPDIR': '/tmp/gnunet/'}, 632 haltOnFailure=True, 633 commands=[ 634 util.ShellArg(command=['./bootstrap'], logname='bootstrap', haltOnFailure=True), 635 util.ShellArg(command=['./configure', 636 "--prefix=/tmp/gnunet-buildbot", 637 "--enable-experimental", 638 "--enable-logging=verbose"], 639 logname='configure', 640 haltOnFailure=True), 641 util.ShellArg(command=['make'], 642 logname='make', 643 haltOnFailure=True), 644 util.ShellArg(command=['make', 'install'], 645 logname='make install', 646 haltOnFailure=True), 647 #util.ShellArg(command=['sudo', '/home/buildbot/bin/netjail.sh', 'bash', '-c', "'make check'"], 648 # logname='make check', 649 # warnOnFailure=True, 650 # flunkOnFailure=False), # make check has issues. 651 util.ShellArg(command=['make', 'uninstall'], 652 logname='make uninstall', 653 haltOnFailure=True)] 654 ) 655 656 gnunet_build_steps = [ 657 steps.Git(repourl='git://git.gnunet.org/gnunet.git', 658 mode='full', method='fresh'), 659 gnunet_make_step 660 ] 661 662 factory.addSteps(gnunet_build_steps) 663 664 ############################ 665 # COVERITY # 666 # Occurs: 1st day of month # 667 ############################ 668 cov_factory = util.BuildFactory() 669 cov_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='fresh')) 670 cov_factory.addStep(steps.ShellSequence( 671 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 672 env={'PATH': "${HOME}/.local/bin:${HOME}/bin:${PATH}"}, 673 commands=[ 674 util.ShellArg(command=['./bootstrap'], logname='bootstrap'), 675 util.ShellArg(command=['./configure', 676 "--prefix=/tmp/gnunet-buildbot", 677 "--enable-experimental=true"], 678 logname="configure"), 679 util.ShellArg(command=['cov-build', '--dir', 'cov-int', 'make'], logname='cov-build'), 680 ])) 681 cov_factory.addStep(steps.ShellCommand(command=['tar', 'czf', 'coverity.tar.gz', 'cov-int/'], 682 haltOnFailure=True, 683 name="Packing up")) 684 cov_factory.addStep(steps.ShellCommand(command=['curl', 685 '--form', util.Interpolate('token=%(secret:coverity_token)s'), 686 '--form', 'email=mschanzenbach@posteo.de', 687 '--form', 'version="git master"', 688 '--form', 'file=@./coverity.tar.gz', 689 '--form', 'description="Buildbot triggered build"', 690 'https://scan.coverity.com/builds?project=GNUnet%2Fgnunet'], 691 haltOnFailure=True, 692 name="Sending")) 693 694 695 ################################################# 696 # RELEASE BUILD # 697 # Occurs: # 698 # 1. Nightly # 699 # 2. Upon a pushed release tag (vX.Y.Z) # 700 # 3. With a commit message containing !tarball # 701 ################################################# 702 dist_factory = util.BuildFactory() 703 # https://github.com/buildbot/buildbot/issues/6539 change to "copy" when fixed 704 dist_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet.git', mode='full', method='clobber')) 705 706 dist_factory.addStep(steps.ShellSequence( 707 name=util.Interpolate("Git rev. %(prop:got_revision)s build"), 708 haltOnFailure=True, 709 commands=[ 710 util.ShellArg(command=['./bootstrap'], 711 logname='bootstrap', 712 haltOnFailure=True), 713 util.ShellArg(command=['./configure'], 714 logname='configure', 715 haltOnFailure=True), 716 util.ShellArg(command=['git', 'status'], 717 logname='status before dist', 718 haltOnFailure=True), 719 util.ShellArg(command=['make', 'dist'], 720 logname="dist", 721 haltOnFailure=True), 722 util.ShellArg(command=['make', 'doxygen'], 723 haltOnFailure=True, 724 logname="Doxygen") 725 ])) 726 727 # Get version number of tarball 728 cmdmeson = r'ls -1 build/meson-dist/gnunet-*.tar.gz | sed "s/build\/meson-dist\/gnunet-//" | sed "s/.tar.gz//" | tail -n1' 729 #cmd = 'git describe --tags | sed "s/^v//"' 730 dist_factory.addStep(steps.SetPropertyFromCommand(hideStepIf=False, 731 command=cmdmeson, 732 property='gnunet_meson_releasever', 733 name="Getting release version")) 734 dist_factory.addStep(steps.ShellCommand(command=["tar", 735 "xf", 736 util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 737 '-C', 'build'], 738 haltOnFailure=True, 739 name="Extracting tarball")) 740 741 # Make doxygen 742 # Try to build dist package 743 dist_factory.addStep(steps.ShellSequence( 744 workdir=util.Interpolate('build/build/gnunet-%(prop:gnunet_meson_releasever)s'), 745 name=util.Interpolate("GNUnet %(prop:gnunet_meson_releasever)s tarball build"), 746 env={'GNUNET_PREFIX': '/tmp/gnunet-buildbot/lib', 747 'PATH': ["/tmp/gnunet-buildbot/bin", "${PATH}"]}, 748 commands=[ 749 util.ShellArg(command=['mkdir', '-p', '$TMPDIR'],logname='tmpdir',haltOnFailure=True), 750 util.ShellArg(command=['./configure', 751 "--prefix=/tmp/gnunet-buildbot", 752 "--enable-experimental=true", 753 "--enable-logging=verbose", 754 "--mesonbuilddir=tarball_build"], 755 logname='setup', 756 haltOnFailure=True), 757 util.ShellArg(command=['make'], 758 logname='compile', 759 haltOnFailure=True), 760 util.ShellArg(command=['make', 'install'], 761 logname='install', 762 haltOnFailure=True), 763 util.ShellArg(command=['make', 'uninstall'], 764 logname='uninstall', 765 haltOnFailure=True)] 766 )) 767 768 # Upload artifact to https://buildbot.gnunet.org/artifacts 769 dist_factory.addStep(steps.FileUpload(workersrc=util.Interpolate('build/meson-dist/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz'), 770 mode=0o644, 771 masterdest=util.Interpolate("~/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"), 772 url=util.Interpolate("https://buildbot.gnunet.org/artifacts/gnunet-%(prop:gnunet_meson_releasever)s.tar.gz"))) 773 774 775 # Update doxygen (TODO skit on nightly?) 776 dist_factory.addStep(steps.ShellSequence( 777 name=util.Interpolate("Deploy Doxygen"), 778 workdir=util.Interpolate('build/doc'), 779 commands=[ 780 util.ShellArg(command=['chmod', '-R', 'ag+rX', '../doc'], 781 logname='Permissions', 782 haltOnFailure=True), 783 util.ShellArg(command=['rsync', '-a', '--delete', 784 '../doc/doxygen', 785 'handbook@firefly.gnunet.org:~/doc_deployment/'], 786 logname='Deploy', 787 haltOnFailure=True), 788 ])) 789 790 791 792 793 ########################### 794 # Fedora COPR build # 795 ########################### 796 copr_factory = util.BuildFactory() 797 copr_factory.addStep(steps.ShellCommand(command=["curl", 798 "-H", "Content-Type: application/json", 799 "--data", "{}", 800 "-X", "POST", 801 "https://copr.fedorainfracloud.org/webhooks/custom/36992/d316c169-1482-4508-a765-cfb5c0efeb67/gnunet/"], 802 haltOnFailure=True, 803 name="Triggering Copr build")) 804 805 806 ########################### 807 # gnunet-gns-registrar # 808 ########################### 809 registrar_factory = util.BuildFactory() 810 registrar_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-gns-registrar.git', mode='incremental')) 811 add_default_pipeline(registrar_factory) 812 813 ########################### 814 # stage.gnunet.org # 815 ########################### 816 www_factory = util.BuildFactory() 817 www_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental')) 818 add_default_pipeline(www_factory) 819 820 ########################### 821 # www.gnunet.org # 822 ########################### 823 www_stable_factory = util.BuildFactory() 824 www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www.git', mode='incremental', branch='stable')) 825 add_default_pipeline(www_stable_factory) 826 827 ########################### 828 # bib.gnunet.org # 829 ########################### 830 bib_factory = util.BuildFactory() 831 bib_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunetbib.git', mode='incremental')) 832 add_default_pipeline(bib_factory) 833 834 835 ########################### 836 # reclaim.gnunet.org # 837 ########################### 838 reclaim_www_stable_factory = util.BuildFactory() 839 reclaim_www_stable_factory.addStep(steps.Git(repourl='git://git.gnunet.org/www-reclaim.git', mode='incremental', branch='master')) 840 add_default_pipeline(reclaim_www_stable_factory) 841 842 843 ########################### 844 # rest.gnunet.org # 845 ########################### 846 rest_factory = util.BuildFactory() 847 rest_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-rest-api.git', mode='incremental', branch='master')) 848 add_default_pipeline(rest_factory) 849 850 ########################### 851 # lsd.gnunet.org # 852 ########################### 853 lsd_factory = util.BuildFactory() 854 lsd_factory.addStep(steps.Git(repourl=util.Property('repository'), mode='full', method="clobber", branch='master')) 855 add_default_pipeline(lsd_factory) 856 857 ########################### 858 # gana.gnunet.org # 859 ########################### 860 gana_factory = util.BuildFactory() 861 gana_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gana.git', mode='full', method="fresh", branch='master')) 862 add_default_pipeline(gana_factory) 863 864 ############################## 865 # doc.gnunet.org (Doc-NG) # 866 ############################## 867 doc_factory = util.BuildFactory() 868 doc_factory.addStep(steps.Git(repourl='git://git.gnunet.org/gnunet-handbook.git', alwaysUseLatest=True, mode='full', method="clobber", branch='master')) 869 add_default_pipeline(doc_factory) 870 871 872 873 BUILDERS.append( 874 util.BuilderConfig(name="gnunet-debian-x86_64", 875 workernames=["firefly-x86_64-amdepyc"], 876 factory=factory)) 877 878 BUILDERS.append( 879 util.BuilderConfig(name="gnunet-fedora-aarch64", 880 workernames=["schanzen-aarch64-fedora-meson"], 881 factory=factory)) 882 883 BUILDERS.append( 884 util.BuilderConfig(name="gnunet-debian-x86_64-dev", 885 workernames=["firefly-x86_64-amdepyc"], 886 factory=factory)) 887 888 BUILDERS.append( 889 util.BuilderConfig(name="gnunet-fedora-aarch64-dev", 890 workernames=["schanzen-aarch64-fedora-meson"], 891 factory=factory)) 892 893 894 BUILDERS.append( 895 util.BuilderConfig(name="gnunet_release", 896 workernames=["firefly-x86_64-amdepyc"], 897 factory=dist_factory)) 898 BUILDERS.append( 899 util.BuilderConfig(name="gnunet_coverity", 900 workernames=["firefly-x86_64-amdepyc"], 901 factory=cov_factory)) 902 903 BUILDERS.append( 904 util.BuilderConfig(name="gnunet-gns-registrar", 905 workernames=["firefly-x86_64-amdepyc", "schanzen-aarch64-fedora-meson"], 906 factory=registrar_factory)) 907 BUILDERS.append( 908 util.BuilderConfig(name="stage.gnunet.org", 909 workernames=["firefly-x86_64-amdepyc"], 910 factory=www_factory)) 911 BUILDERS.append( 912 util.BuilderConfig(name="www.gnunet.org", 913 workernames=["firefly-x86_64-amdepyc"], 914 factory=www_stable_factory)) 915 BUILDERS.append( 916 util.BuilderConfig(name="bib.gnunet.org", 917 workernames=["firefly-x86_64-amdepyc"], 918 factory=bib_factory)) 919 BUILDERS.append( 920 util.BuilderConfig(name="reclaim.gnunet.org", 921 workernames=["firefly-x86_64-amdepyc"], 922 factory=reclaim_www_stable_factory)) 923 BUILDERS.append( 924 util.BuilderConfig(name="rest.gnunet.org", 925 workernames=["firefly-x86_64-amdepyc"], 926 factory=rest_factory)) 927 BUILDERS.append( 928 util.BuilderConfig(name="lsd.gnunet.org", 929 workernames=["firefly-x86_64-amdepyc"], 930 factory=lsd_factory)) 931 BUILDERS.append( 932 util.BuilderConfig(name="gana.gnunet.org", 933 workernames=["firefly-x86_64-amdepyc"], 934 factory=gana_factory)) 935 BUILDERS.append( 936 util.BuilderConfig(name="doc.gnunet.org", 937 workernames=["firefly-x86_64-amdepyc"], 938 factory=doc_factory)) 939 BUILDERS.append( 940 util.BuilderConfig(name="gnunet_rpm_copr", 941 workernames=["firefly-x86_64-amdepyc"], 942 factory=copr_factory)) 943 944 945 bsg = reporters.BuildStatusGenerator(mode=["exception", "failing", "problem"], 946 builders=["gnunet-debian-x86_64", "gnunet-fedora-aarch64"]) 947 948 mn = reporters.MailNotifier(fromaddr="buildbot@firefly.gnunet.org", 949 sendToInterestedUsers=False, 950 extraRecipients=["gnunet-ci@gnunet.org"], 951 lookup="gnunet.org", 952 generators=[bsg]) 953 SERVICES.append(mn) 954 955 956 957 958 ############################################# 959 # 19: CONTAINER FACTORY ##################### 960 ############################################# 961 ## 962 # These factories uses the standard container worker. 963 WORKERS.append(Worker("container-worker", "container-pass")) 964 965 966 # Container Job Generator Functions 967 # Return the standard configuration for a container job. 968 def default_job_config(jobName, repoName): 969 return {jobName: {"HALT_ON_FAILURE": True, 970 "WARN_ON_FAILURE": False, 971 "CONTAINER_BUILD": True, 972 "CONTAINER_NAME": repoName, 973 "CONTAINER_ARCH": "amd64", 974 # Which schedulers a job is for: "push", "nightly" or 975 # "both". The default keeps every existing job exactly 976 # where it was -- stages are discovered by listing 977 # contrib/ci/jobs, so a job that said nothing used to 978 # run in every build of its repository, and a repository 979 # without a nightly scheduler still has only one kind. 980 "RUN_ON": "push"}} 981 982 983 # Parse a job config and apply it as overrides to the standard configuration. 984 def ingest_job_config(configText, jobName, repoName): 985 configDict = default_job_config(jobName, repoName) 986 jobIni = configparser.ConfigParser() 987 jobIni.optionxform = str 988 jobIni.read_string(configText) 989 configDict[jobName].update(jobIni["build"]) 990 print(configDict) 991 return configDict 992 993 994 # Search for configs, and ingest 995 def handle_job_config(jobName, repoName, configText, configResult): 996 print(configText) 997 if configResult == util.SUCCESS: 998 print(f"Ingesting Job Config: {configText}") 999 configDict = ingest_job_config(configText, jobName, repoName) 1000 print(configDict) 1001 return configDict 1002 else: 1003 print("No job config; Using default params") 1004 return default_job_config(jobName, repoName) 1005 1006 1007 class GenerateStagesCommand(buildstep.ShellMixin, steps.BuildStep): 1008 1009 def __init__(self, REPO_NAME, **kwargs): 1010 self.REPO_NAME = REPO_NAME 1011 kwargs = self.setupShellMixin(kwargs) 1012 super().__init__(**kwargs) 1013 self.observer = logobserver.BufferLogObserver() 1014 self.addLogObserver('stdio', self.observer) 1015 1016 def extract_stages(self, stdout): 1017 stages = [] 1018 for line in stdout.split('\n'): 1019 stage = str(line.strip()) 1020 if stage: 1021 stages.append(stage) 1022 return stages 1023 1024 @defer.inlineCallbacks 1025 def run(self): 1026 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{self.REPO_NAME}" 1027 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1028 # run 'ls <project_root>/contrib/ci/jobs/' to get the list of stages 1029 cmd = yield self.makeRemoteShellCommand() 1030 yield self.runCommand(cmd) 1031 jobDirs = [] 1032 1033 # if the command passes extract the list of stages 1034 result = cmd.results() 1035 if result == util.SUCCESS: 1036 jobDirs = self.extract_stages(self.observer.getStdout()) 1037 print(f"this is jobDirs list: {jobDirs}") 1038 self.configDict = {} 1039 print(f"Remote cmd stdout: {self.observer.getStdout()}") 1040 print(f"cmd.results: {cmd.results()}") 1041 for stage in jobDirs: 1042 jobDirPath = f"{CI_JOBS_PATH}/{stage}" 1043 observer = logobserver.BufferLogObserver() 1044 self.addLogObserver('stdio', observer) 1045 cmd1 = yield self.makeRemoteShellCommand( 1046 command=["cat", f"{jobDirPath}/config.ini"]) 1047 yield self.runCommand(cmd1) 1048 print(f"cmd1.results: {cmd1.results()}") 1049 print(f"Second command stdout: {observer.getStdout()}") 1050 print(f"Current stage: {stage}") 1051 print(jobDirPath) 1052 self.configDict.update( 1053 handle_job_config( 1054 stage, self.REPO_NAME, 1055 observer.getStdout(), cmd1.results())) 1056 print(self.configDict) 1057 # create a container step for each stage and 1058 # add them to the build 1059 convstr2bool = ast.literal_eval 1060 self.build.addStepsAfterCurrentStep([ 1061 container_add_step( 1062 convstr2bool( 1063 str(self.configDict[stage]["HALT_ON_FAILURE"])), 1064 convstr2bool( 1065 str(self.configDict[stage]["WARN_ON_FAILURE"])), 1066 convstr2bool( 1067 str(self.configDict[stage]["CONTAINER_BUILD"])), 1068 self.configDict[stage]["CONTAINER_NAME"], 1069 container_factory, 1070 CONTAINER_WORKDIR, 1071 self.REPO_NAME, 1072 stage, 1073 self.configDict[stage]["CONTAINER_ARCH"], 1074 f"contrib/ci/jobs/{stage}/job.sh", 1075 RUN_ON=self.configDict[stage]["RUN_ON"]) 1076 for stage in jobDirs 1077 ]) 1078 1079 return result 1080 1081 # MHD2 uses a deployment-owned Debian overlay and a reviewed upstream pin. 1082 # Both architectures must pass before either is uploaded to the nightly feed. 1083 MHD2_WORKDIR = "/home/container-worker/workspace/libmicrohttpd2-ci" 1084 mhd2_factory = util.BuildFactory() 1085 mhd2_factory.addStep(Git( 1086 name="git", repourl="git://git.taler.net/taler-deployment.git", 1087 branch="master", mode="full", method="fresh", workdir=MHD2_WORKDIR, 1088 haltOnFailure=True)) 1089 for arch in ("amd64", "arm64"): 1090 mhd2_factory.addStep(container_add_step( 1091 True, False, True, "unused", mhd2_factory, MHD2_WORKDIR, 1092 "libmicrohttpd2", f"build-{arch}", arch, 1093 "buildbot/libmicrohttpd2/build.sh", 1094 "buildbot/libmicrohttpd2/Containerfile")) 1095 mhd2_factory.addStep(container_add_step( 1096 True, False, False, "localhost/buildbot/libmicrohttpd2/build-amd64:amd64", 1097 mhd2_factory, MHD2_WORKDIR, "libmicrohttpd2", "upload", "amd64", 1098 "buildbot/libmicrohttpd2/upload.sh")) 1099 BUILDERS.append(util.BuilderConfig( 1100 name="libmicrohttpd2-builder", workernames=["container-worker"], 1101 factory=mhd2_factory)) 1102 1103 1104 def mhd2_packaging_changed(change): 1105 return any(path == "buildbot/master.cfg" or path.startswith(( 1106 "buildbot/libmicrohttpd2/", 1107 "packaging/ng/buildscripts/debian-overlays/libmicrohttpd2/")) 1108 for path in change.files) 1109 1110 1111 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1112 name="libmicrohttpd2-packaging-scheduler", 1113 change_filter=util.ChangeFilter(branch="master", project_re="^taler-deployment$"), 1114 fileIsImportant=mhd2_packaging_changed, treeStableTimer=30, 1115 builderNames=["libmicrohttpd2-builder"])) 1116 1117 # List of repos to add to container factory. 1118 container_repos = ["git.gnunet.org/gnunet", 1119 "git.taler.net/anastasis", 1120 "git.taler.net/challenger", 1121 "git.taler.net/donau", 1122 "git.taler.net/exchange", 1123 "git.taler.net/libeufin", 1124 "git.taler.net/taler-rust", 1125 "git.taler.net/depolymerization", 1126 "git.taler.net/merchant", 1127 "git.taler.net/sync", 1128 "git.taler.net/taler-android", 1129 "git.taler.net/taler-mailbox", 1130 "git.taler.net/taldir", 1131 "git.taler.net/taler-typescript-core",] 1132 1133 # Repositories with at least one `RUN_ON = nightly` job in contrib/ci/jobs. 1134 # Listed rather than inferred because the stage list is only known once a 1135 # build is running, while a scheduler has to exist before that: a nightly for 1136 # a repository without such a job would be a build whose every stage skips. 1137 nightly_container_repos = ["taler-android"] 1138 1139 for repo in container_repos: 1140 1141 # Factory-wide variables 1142 REPO_NAME = repo.rsplit('/', 1)[1] 1143 REPO_URL = "git://" + repo + ".git" 1144 CACHE_CHECKOUT = REPO_NAME in ( 1145 "gnunet", "exchange", "merchant", "donau", "challenger", "anastasis", 1146 "sync", "libeufin", "taler-rust", "depolymerization", "taler-mailbox", 1147 "taldir", "taler-typescript-core", 1148 ) 1149 CONTAINER_WORKDIR = f"/home/container-worker/workspace/{REPO_NAME}" 1150 CI_JOBS_PATH = f"{CONTAINER_WORKDIR}/contrib/ci/jobs" 1151 1152 # Create a factory 1153 container_factory = util.BuildFactory() 1154 container_factory.workdir = CONTAINER_WORKDIR 1155 1156 # Containers can leave files owned by root. Make them writable before cleanup. 1157 workspace_command = ( 1158 f"if test -d {CONTAINER_WORKDIR}; then " 1159 f"podman run --log-driver=none --rm --volume {CONTAINER_WORKDIR}:/workdir " 1160 "docker.io/library/debian:bookworm-slim chmod -R 777 /workdir; fi" 1161 ) 1162 if CACHE_CHECKOUT: 1163 # Replace the previous shallow clone once, then retain the full checkout. 1164 workspace_command += ( 1165 f" && if test -f {CONTAINER_WORKDIR}/.git/shallow; then " 1166 f"rm -rf {CONTAINER_WORKDIR}; fi" 1167 ) 1168 else: 1169 workspace_command += f" && rm -rf {CONTAINER_WORKDIR}" 1170 workspace_command += f" && mkdir -p {CONTAINER_WORKDIR}" 1171 1172 container_factory.addStep(ShellCommand( 1173 name="workspace", 1174 descriptionDone="Workspace directory check", 1175 command=workspace_command, 1176 haltOnFailure=True, 1177 )) 1178 1179 # "fresh" cleans build outputs but retains Git history between builds. 1180 # Debian package versions require full history and all release tags. 1181 # Other container builders use a new depth-1 clone on every build. 1182 # Will checkout value of "branch" property from job properties. 1183 # https://docs.buildbot.net/latest/manual/configuration/steps/source_git.html 1184 container_factory.addStep(Git( 1185 name="git", 1186 repourl=REPO_URL, 1187 branch=util.Interpolate('%(src::branch)s'), 1188 mode='full', 1189 method='fresh' if CACHE_CHECKOUT else 'clobber', 1190 shallow=not CACHE_CHECKOUT, 1191 tags=CACHE_CHECKOUT, 1192 submodules=True, 1193 haltOnFailure=True, 1194 )) 1195 1196 container_factory.addStep(GenerateStagesCommand( 1197 REPO_NAME, 1198 name="Generate build stages", 1199 command=f"ls {CI_JOBS_PATH}", 1200 haltOnFailure=True)) 1201 1202 BUILDERS.append(util.BuilderConfig( 1203 name=f"{REPO_NAME}-builder", 1204 workernames=["container-worker"], 1205 factory=container_factory 1206 )) 1207 1208 # Only enable this scheduler for debugging! 1209 # Will run builders with 1 minute of waiting inbetween builds 1210 # SCHEDULERS.append(schedulers.Periodic( 1211 # name=f"{REPO_NAME}-minutely", 1212 # builderNames=[f"{REPO_NAME}-builder"], 1213 # periodicBuildTimer=60 1214 # )) 1215 1216 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1217 name=f"{REPO_NAME}-container-scheduler", 1218 change_filter=util.ChangeFilter( 1219 branch="master", 1220 # Anchored: ChangeFilter matches the regex against the start of 1221 # the project name only, so an unanchored "(anastasis)" would 1222 # also fire for pushes to 'anastasis-www' and 'anastasis-gtk'. 1223 project_re=f"^({REPO_NAME})$" 1224 ), 1225 treeStableTimer=30, 1226 builderNames=[f"{REPO_NAME}-builder"] 1227 )) 1228 1229 # Jobs whose config.ini says `RUN_ON = nightly` run from here instead of 1230 # on every push: the android integration suite drives a real emulator and 1231 # takes about three hours, which is not something to put in front of the 1232 # next push to a worker shared with every other repository. 1233 # 1234 # onlyIfChanged keeps a quiet day quiet -- no change to this repository 1235 # since the last build means no build tonight. Note that changes arrive 1236 # here from the git hook (PBChangeSource), not from a poller, so a 1237 # repository whose hook is silent gets no nightly rather than a redundant 1238 # one. 1239 # 1240 # The builder is the same one the push scheduler uses; what differs is the 1241 # `scheduler` property, which is how container_add_step decides whether a 1242 # given stage belongs in this build. 1243 if REPO_NAME in nightly_container_repos: 1244 SCHEDULERS.append(schedulers.Nightly( 1245 name=f"{REPO_NAME}-nightly", 1246 change_filter=util.ChangeFilter( 1247 branch="master", 1248 project_re=f"^({REPO_NAME})$" 1249 ), 1250 builderNames=[f"{REPO_NAME}-builder"], 1251 onlyIfChanged=True, 1252 hour=2, minute=30)) 1253 1254 SERVICES.append(reporters.MailNotifier( 1255 fromaddr="buildbot@taler.net", 1256 # notify from pass to fail, and viceversa. 1257 generators=[BuildStatusGenerator( 1258 mode=('change','problem','failing','exception',), 1259 builders=[f"{REPO_NAME}-builder",], 1260 message_formatter=reporters.MessageFormatter( 1261 template_type='plain', 1262 want_logs_content=True, 1263 ), 1264 )], 1265 sendToInterestedUsers=False, 1266 useTls=False, 1267 relayhost="localhost", 1268 smtpPort=25, 1269 dumpMailsToLog=True, 1270 extraRecipients=[f"ci-{REPO_NAME}@taler.net"] 1271 )) 1272 1273 1274 ################ 99: debug stuff JOB ################################### 1275 1276 # This does nothing, just a starting point for a factory. 1277 DEBUG_FACTORY = util.BuildFactory() 1278 DEBUG_FACTORY.addStep( 1279 ShellCommand( 1280 name="echo debug", 1281 description="just echoing a word", 1282 descriptionDone="builder responded", 1283 command=["echo", "I'm here!"] 1284 ) 1285 ) 1286 1287 1288 ################################################################## 1289 #################### General purpose ############################# 1290 ################################################################## 1291 1292 # Compute array of the names of all of our builders 1293 BUILDER_LIST = map(lambda builder: builder.name, BUILDERS) 1294 1295 ####### GENERAL PURPOSE BUILDBOT SERVICES ####################### 1296 1297 SERVICES.append(reporters.MailNotifier( 1298 fromaddr="testbuild@taler.net", 1299 # notify from pass to fail, and viceversa. 1300 generators=[BuildStatusGenerator( 1301 mode=('change','problem','failing','exception',), 1302 builders=EMAIL_ALERTS, 1303 message_formatter=reporters.MessageFormatter( 1304 template_type='plain', 1305 want_logs_content=True, 1306 ), 1307 )], 1308 sendToInterestedUsers=False, 1309 useTls=False, 1310 relayhost="localhost", 1311 smtpPort=25, 1312 dumpMailsToLog=True, 1313 extraRecipients=BUILDER_EMAIL_ADDRESSES 1314 )) 1315 1316 1317 ############# GENERAL PURPOSE SCHEDULERS ########################## 1318 1319 # Workers that are done on wallet or deployment changes to master 1320 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1321 name="taler-healthcheck-scheduler", 1322 change_filter=util.ChangeFilter( 1323 branch="master", 1324 project_re="(taler-typescript-core|taler-deployment)" 1325 ), 1326 treeStableTimer=None, 1327 builderNames=WALLETCHANGE_TRIGGERS 1328 )) 1329 1330 SCHEDULERS.append(schedulers.SingleBranchScheduler( 1331 name="all-scheduler", 1332 change_filter=util.ChangeFilter( 1333 branch_re="(master|stable)", 1334 project_re="(taler-typescript-core|exchange|" 1335 "merchant|taler-deployment|twister|sync|" 1336 "taler-merchant-demos)" 1337 ), 1338 treeStableTimer=None, 1339 builderNames=CODECHANGE_TRIGGERS 1340 )) 1341 1342 # Scheduler for all nightly builds. 1343 SCHEDULERS.append(schedulers.Nightly( 1344 name="nightly-scheduler", 1345 builderNames=list(NIGHTLY_TRIGGERS), 1346 branch="master", 1347 hour=6, 1348 minute=0 1349 )) 1350 1351 # Provide "force" button in the web UI. 1352 SCHEDULERS.append(schedulers.ForceScheduler( 1353 name="force-scheduler", 1354 buttonName="Force build", 1355 builderNames=list(BUILDER_LIST) 1356 )) 1357 1358 ######################################################### 1359 ####### Actual configuation initialization ############## 1360 ######################################################### 1361 1362 # This is the dictionary that the buildmaster pays attention to. We also use 1363 # a shorter alias to save typing. 1364 c = BuildmasterConfig = {} 1365 1366 # Secrets 1367 c['secretsProviders'] = [secrets.SecretInAFile(dirname="/home/buildbot-master/secrets")] 1368 1369 c["workers"] = WORKERS 1370 c["builders"] = BUILDERS 1371 c["schedulers"] = SCHEDULERS 1372 c["services"] = SERVICES 1373 1374 # Silence warning and allow very basic phoning home. 1375 c["buildbotNetUsageData"] = "basic" 1376 1377 c["title"] = "GNUnet Builder" 1378 c["titleURL"] = "https://buildbot.gnunet.org" 1379 1380 # the 'buildbotURL' string should point to the location where the buildbot's 1381 # internal web server is visible. 1382 c['buildbotURL'] = "https://buildbot.gnunet.org/" 1383 1384 # This specifies what database buildbot uses to store its 1385 # state. You can leave this at its default for all but the 1386 # largest installations. 1387 c["db"] = { 1388 "db_url": "sqlite:///state.sqlite", 1389 } 1390 1391 # the 'change_source' setting tells the buildmaster how it should 1392 # find out about source code changes. 1393 pbSource = changes.PBChangeSource(port="tcp:19990:interface=127.0.0.1", 1394 user="git-buildbot", 1395 passwd="Aer3eari") 1396 1397 pollGnunetSource = changes.GitPoller(repourl='git://git.gnunet.org/gnunet.git', 1398 branches=True, 1399 pollInterval=300, 1400 pollAtLaunch=True, 1401 project="gnunet") 1402 c["change_source"] = [pollGnunetSource, pbSource] 1403 1404 # 'protocols' contains information about protocols which master 1405 # will use for communicating with workers. You must define at 1406 # least 'port' option that workers could connect to your master 1407 # with this protocol. 'port' must match the value configured into 1408 # the workers (with their --master option) 1409 c["protocols"] = {"pb": {"port": "tcp:19989"}} 1410 1411 # Load admin password 1412 with open("/home/buildbot-master/secrets/admin") as fh: 1413 www_pass = fh.read().strip() 1414 1415 # minimalistic config to activate new web UI 1416 # -- formerly commented out as not packaged properly in Debian and others, see 1417 # https://bugzilla.redhat.com/show_bug.cgi?id=1557687 1418 c["www"] = { 1419 "port": "tcp:8009:interface=127.0.0.1", 1420 "default_page": 'builders', 1421 "plugins": { 1422 "waterfall_view": True, 1423 "console_view": True, 1424 "grid_view": True, 1425 }, 1426 "auth": util.UserPasswordAuth([('admin',www_pass)]), 1427 "authz": util.Authz( 1428 allowRules=[ 1429 # Grants control rights (force, stop, rebuild) exclusively to the 'admins' role 1430 util.AnyControlEndpointMatcher(role="admins") 1431 ], 1432 roleMatchers=[ 1433 # Maps 'admin' user profile to the 'admins' role 1434 util.RolesFromUsername(roles=["admins"], usernames=["admin"]) 1435 ] 1436 ), 1437 "allowed_origins": ["https://*.taler.net","https://*.gnunet.org"], 1438 "avatar_methods": [], 1439 }