taler-pkg (24023B)
1 #!/usr/bin/env python3 2 3 # Copyright (c) 2024 Taler Systems SA 4 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 5 # SPDX-License-Identifier: GPL-3.0-or-later 6 7 import argparse 8 import subprocess 9 import platform 10 import os 11 import re 12 import sys 13 from pathlib import Path 14 15 # Make local util package available 16 file = Path(__file__).resolve() 17 parent, root = file.parent, file.parents[1] 18 sys.path.append(str(root)) 19 20 from util import vercomp 21 22 mydir = os.path.dirname(os.path.realpath(__file__)) 23 24 archs = ["arm64", "amd64"] 25 host = "taler.net" 26 native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" 27 28 components = [ 29 "taler-wallet-cli", 30 "taler-merchant-webui", 31 "taler-exchange-kyc-webui", 32 "taler-exchange-aml-webui", 33 "taler-auditor-webui", 34 "taler-challenger-helpers", 35 "challenger-webui", 36 "gnunet", 37 "libeufin", 38 "donau", 39 "paivana-httpd", 40 "challenger", 41 "taler-exchange", 42 "taler-harness", 43 "taler-merchant", 44 "taler-rust", 45 "robocop", 46 #"depolymerization", 47 # These two packages don't have good debs yet, 48 # Debian complains "No section given for ..., skipping. 49 # "taler-directory", 50 # "taler-mailbox", 51 # We don't publish packages for these yet 52 # "taler-mdb", 53 # "taler-merchant-demos", 54 "anastasis", 55 "anastasis-gtk", 56 # Currently not used anywhere 57 # "sync", 58 ] 59 60 deps = { 61 "taler-exchange": ["gnunet"], 62 "anastasis": ["gnunet", "taler-merchant"], 63 "anastasis-gtk": ["anastasis"], 64 "taler-merchant": ["gnunet", "taler-exchange", "donau"], 65 "donau": ["gnunet", "taler-exchange"], 66 "challenger": ["taler-exchange"], 67 # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"], 68 "sync": ["taler-merchant", "taler-exchange", "gnunet"], 69 "paivana-httpd": ["taler-merchant", "taler-exchange", "gnunet"], 70 } 71 72 # Compute reverse dependencies 73 rdeps = {} 74 for n1, d in deps.items(): 75 for n2 in d: 76 rd = rdeps.setdefault(n2, []) 77 if n1 not in rd: 78 rd.append(n1) 79 80 81 def buildsort(roots): 82 """Toposort transitive closure of roots based on deps""" 83 out = [] 84 stack = list(roots[::-1]) 85 pmark = set() 86 tmark = set() 87 while len(stack): 88 node = stack[-1] 89 if node in pmark: 90 stack.pop() 91 tmark.discard(node) 92 continue 93 done = True 94 for dep in deps.get(node, []): 95 if dep not in pmark: 96 if dep in tmark: 97 raise Exception("cycle") 98 stack.append(dep) 99 tmark.add(node) 100 done = False 101 if done: 102 pmark.add(node) 103 out.append(node) 104 stack.pop() 105 tmark.discard(node) 106 return out 107 108 109 def propagate_outdated(outdated): 110 """Propagate outdatedness to dependees""" 111 closure = set() 112 q = list(outdated) 113 while len(q): 114 n = q.pop() 115 closure.add(n) 116 for r in rdeps.get(n, []): 117 if r not in closure: 118 closure.add(r) 119 q.append(r) 120 return closure 121 122 123 def find_outdated(pkgdir, arch, roots): 124 """Find outdated components based on tag files""" 125 outdated = set() 126 for component in roots: 127 ver_requested = open(f"buildconfig/{component}.tag").read().strip() 128 built_tag_file = pkgdir / f"{component}@{arch}.built.tag" 129 ver_built = None 130 if built_tag_file.exists(): 131 ver_built = open(built_tag_file).read().strip() 132 if ver_built != ver_requested: 133 outdated.add(component) 134 print(component, ver_built, "->", ver_requested) 135 return outdated 136 137 138 def build(cfg): 139 transitive = False 140 if cfg.transitive: 141 transitive = True 142 distro = cfg.distro 143 vendor, codename = distro.split("-", 1) 144 print("building", distro) 145 dockerfile = f"distros/{distro}.Dockerfile" 146 image_tag = f"localhost/taler-packaging-{distro}:latest" 147 pkgdir = Path(f"packages/{distro}").absolute() 148 cachedir = Path("cache").absolute() 149 cachedir.mkdir(exist_ok=True) 150 (cachedir / "cargo-git").mkdir(exist_ok=True) 151 (cachedir / "cargo-registry").mkdir(exist_ok=True) 152 (cachedir / "cargo-build").mkdir(exist_ok=True) 153 (cachedir / "gradle").mkdir(exist_ok=True) 154 (cachedir / "pnpm").mkdir(exist_ok=True) 155 (cachedir / distro / "apt-archives").mkdir(parents=True, exist_ok=True) 156 (cachedir / distro / "apt-lists").mkdir(parents=True, exist_ok=True) 157 158 if cfg.arch is None: 159 arch_list = [native_arch] 160 else: 161 arch_list = cfg.arch.split(",") 162 163 if not cfg.dry: 164 for arch in arch_list: 165 subprocess.run( 166 [ 167 "podman", 168 "build", 169 "--arch", 170 arch, 171 "-v", 172 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 173 "-v", 174 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 175 "-t", 176 image_tag, 177 "-f", 178 dockerfile, 179 ], 180 check=True, 181 ) 182 183 # Sort components by their dependencies 184 buildorder = buildsort(components) 185 print("build order:", buildorder) 186 187 for arch in arch_list: 188 outdated = find_outdated(pkgdir, arch, buildorder) 189 190 # Propagate outdatedness to dependees 191 closure = propagate_outdated(outdated) 192 193 print("outdated closure", closure) 194 195 for component in buildorder: 196 if transitive: 197 if component not in closure: 198 continue 199 else: 200 if component not in outdated: 201 continue 202 print("building", component) 203 pkgdir.mkdir(parents=True, exist_ok=True) 204 cmd = [ 205 "podman", 206 "run", 207 "-it", 208 "--arch", 209 arch, 210 "--entrypoint=/bin/python3", 211 "--security-opt", 212 "label=disable", 213 "--mount", 214 f"type=bind,source={cachedir}/gradle,target=/root/.gradle/caches", 215 "--mount", 216 f"type=bind,source={cachedir}/pnpm,target=/root/.local/share/pnpm/store", 217 "--mount", 218 f"type=bind,source={cachedir}/cargo-registry,target=/root/.cargo/registry", 219 "--mount", 220 f"type=bind,source={cachedir}/cargo-git,target=/root/.cargo/git", 221 "--mount", 222 f"type=bind,source={cachedir}/cargo-build,target=/root/.cargo-build", 223 "--env", 224 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 225 "--mount", 226 f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared", 227 "--mount", 228 f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared", 229 "--mount", 230 f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", 231 "--mount", 232 f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly", 233 "--mount", 234 f"type=bind,source={pkgdir},target=/pkgdir", 235 image_tag, 236 "/buildscripts/generic", 237 component, 238 codename, 239 arch, 240 ] 241 if not cfg.dry: 242 subprocess.run( 243 cmd, 244 check=True, 245 ) 246 247 248 def show_order(cfg): 249 buildorder = buildsort(list(cfg.roots)) 250 print("build order:", buildorder) 251 252 253 254 255 def promote(cfg): 256 dry = cfg.dry 257 distro = cfg.distro 258 vendor, codename = distro.split("-", 1) 259 listfmt = "${package}_${version}_${architecture}.${$type}\n" 260 if dry: 261 subprocess.run( 262 [ 263 "ssh", 264 f"taler-packaging@{host}", 265 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}", 266 ], 267 check=True, 268 ) 269 else: 270 subprocess.run( 271 [ 272 "ssh", 273 "-t", 274 f"taler-packaging@{host}", 275 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}", 276 ], 277 check=True, 278 ) 279 # Always export! 280 # Reprepro is weird, listed packages might actually not show 281 # up in the index yet. 282 subprocess.run( 283 [ 284 "ssh", 285 "-t", 286 f"taler-packaging@{host}", 287 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}", 288 ], 289 check=True, 290 ) 291 292 def show_published(cfg): 293 distro = cfg.distro 294 vendor, codename = distro.split("-", 1) 295 listfmt = "${package}_${version}_${architecture}.${$type}\n" 296 subprocess.run( 297 [ 298 "ssh", 299 f"taler-packaging@{host}", 300 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 301 ], 302 check=True, 303 ) 304 305 306 def test(cfg): 307 target = cfg.distro 308 vendor, codename, *rest = target.split("-") 309 distro = f"{vendor}-{codename}" 310 image_tag = f"localhost/taler-packaging-{distro}:latest" 311 dockerfile = f"distros/{distro}.Dockerfile" 312 cachedir = Path(f"cache").absolute() 313 print("building base image") 314 subprocess.run( 315 [ 316 "podman", 317 "build", 318 "-v", 319 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 320 "-v", 321 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 322 "-t", 323 image_tag, 324 "-f", 325 dockerfile, 326 ], 327 check=True, 328 ) 329 print("running test") 330 cmd = [ 331 "podman", 332 "run", 333 "-it", 334 "--entrypoint=/bin/bash", 335 "--security-opt", 336 "label=disable", 337 "--mount", 338 f"type=bind,source={mydir}/testing,target=/testing,readonly", 339 image_tag, 340 f"/testing/test-{target}", 341 ] 342 subprocess.run( 343 cmd, 344 check=True, 345 ) 346 347 def publish(cfg): 348 distro = cfg.distro 349 if distro.endswith("-testing"): 350 print("Files are automatically published to testing", file=sys.stderr) 351 sys.exit(1) 352 vendor, codename = distro.split("-", 1) 353 # List of .deb and .ddeb files. 354 debs = [] 355 listfmt = "${package}_${version}_${architecture}.${$type}\n" 356 server_debs_str = subprocess.check_output( 357 [ 358 "ssh", 359 f"taler-packaging@{host}", 360 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 361 ], 362 encoding="utf-8", 363 ) 364 server_debs = server_debs_str.split() 365 for component in components: 366 current = [] 367 for arch in archs + ["all"]: 368 cf = Path(f"./packages/{distro}/{component}@{arch}.built.current") 369 if not cf.exists(): 370 print(f"component {component}@{arch} has no current packages") 371 continue 372 with open(cf) as f: 373 current = current + f.read().split() 374 print("current", current) 375 for deb in current: 376 if deb.endswith(".deb"): 377 pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_") 378 elif deb.endswith(".ddeb"): 379 pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_") 380 else: 381 raise Error(f"invalid deb filename: {deb}") 382 fresh = True 383 server_deb = None 384 # If the server has the same or a later version, 385 # the local version isn't fresh. 386 for srvdeb in server_debs: 387 pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_") 388 if pkg1 != pkg2 or arch1 != arch2: 389 continue 390 if vercomp.compare_versions(ver1, ver2) <= 0: 391 fresh = False 392 server_deb = srvdeb 393 break 394 if fresh: 395 debs.append(deb) 396 else: 397 print("package", deb, "not fresh, server has", server_deb) 398 if len(debs) == 0: 399 print("nothing to upload") 400 else: 401 print("uploading debs", debs) 402 if cfg.dry: 403 return 404 debs = [Path(f"./packages/{distro}/") / x for x in debs] 405 subprocess.run( 406 [ 407 "ssh", 408 f"taler-packaging@{host}", 409 f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb", 410 ], 411 check=True, 412 ) 413 subprocess.run( 414 ["rsync", "-a", "--info=progress2", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True 415 ) 416 ret = subprocess.run( 417 [ 418 "ssh", 419 "-t", 420 f"taler-packaging@{host}", 421 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb", 422 ], 423 ) 424 if ret.returncode != 0: 425 # Usually not critical if it fails. 426 print( 427 "Including ddebs failed. This can happen when including packages that have been included previously" 428 ) 429 # Almost the same, but with ddebs. 430 # We explicitly need to tell reprepro 431 # to ignore the extension, because it does not 432 # deal well with ddebs out of the box. 433 ret = subprocess.run( 434 [ 435 "ssh", 436 "-t", 437 f"taler-packaging@{host}", 438 f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb", 439 ], 440 ) 441 if ret.returncode != 0: 442 # Usually not critical if it fails. 443 print( 444 "Including ddebs failed. This can happen when including packages that have been included previously" 445 ) 446 # Always export! 447 # Reprepro is weird, listed packages might actually not show 448 # up in the index yet. 449 subprocess.run( 450 [ 451 "ssh", 452 "-t", 453 f"taler-packaging@{host}", 454 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 455 ], 456 check=True, 457 ) 458 459 460 def get_remote_version(url): 461 """Get the latest stable tag from the git repo""" 462 # Construct the git command 463 # We use -c versionsort.suffix=- to ensure correct semantic version sorting 464 cmd = [ 465 "git", 466 "-c", 467 "versionsort.suffix=-", 468 "ls-remote", 469 "--exit-code", 470 "--refs", 471 "--sort=version:refname", 472 "--tags", 473 url, 474 "*.*.*", 475 ] 476 477 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 478 479 # Parse the output 480 # Output format is usually: <hash>\trefs/tags/<tagname> 481 lines = result.stdout.strip().split("\n") 482 483 valid_tags = [] 484 485 for line in lines: 486 parts = line.split() 487 if len(parts) < 2: 488 continue 489 490 # refs/tags/v1.0.0 -> v1.0.0 491 ref_path = parts[1] 492 tag = ref_path.split("/")[-1] 493 494 # Exclude pre-release semver versions 495 if tag.startswith("v") and "-" in tag: 496 continue 497 498 valid_tags.append(tag) 499 500 if valid_tags: 501 return valid_tags[-1] 502 return "(none)" 503 504 505 def check_version(name, url): 506 """ 507 Compares local buildconfig version with remote git version. 508 """ 509 ver = get_remote_version(url) 510 config_path = os.path.join("buildconfig", f"{name}.tag") 511 with open(config_path, "r") as f: 512 curr = f.read().strip() 513 prefix = "[!] " if curr != ver else "" 514 print(f"{prefix}{name} curr: {curr} latest: {ver}") 515 516 517 def print_latest(cfg): 518 """Print latest upstream tag for each component""" 519 for name in components: 520 config_path = os.path.join("buildconfig", f"{name}.giturl") 521 with open(config_path, "r") as f: 522 giturl = f.read().strip() 523 check_version(name, giturl) 524 525 526 # Tag syntax variants supported by buildscripts/generic: 527 # v$maj.$min.$patch => release version 528 # v$maj.$min.$patch-dev.$n => dev version 529 # deb-v$maj.$min.$patch-$revision => release version with debian revision 530 # Debian revisions of dev versions are *not* supported 531 tag_re_release = re.compile(r"v(\d+)\.(\d+)\.(\d+)") 532 tag_re_dev = re.compile(r"v(\d+)\.(\d+)\.(\d+)-dev\.(\d+)") 533 tag_re_deb = re.compile(r"deb-v(\d+)\.(\d+)\.(\d+)(?:-(\d+))?") 534 535 536 def tag_sortkey(tag): 537 """Get a sort key for a tag, or None if the tag syntax isn't supported. 538 539 Dev versions sort before the corresponding release version, debian 540 revisions sort after it. 541 """ 542 m = tag_re_release.fullmatch(tag) 543 if m: 544 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, 0) 545 m = tag_re_dev.fullmatch(tag) 546 if m: 547 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 0, int(m.group(4))) 548 m = tag_re_deb.fullmatch(tag) 549 if m: 550 rev = m.group(4) 551 return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, int(rev or 0)) 552 return None 553 554 555 def list_remote_tags(url): 556 """Get all tags from the git repo""" 557 cmd = ["git", "ls-remote", "--exit-code", "--refs", "--tags", url] 558 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 559 tags = [] 560 for line in result.stdout.strip().split("\n"): 561 parts = line.split() 562 if len(parts) < 2: 563 continue 564 # refs/tags/v1.0.0 -> v1.0.0 565 tags.append(parts[1].split("/")[-1]) 566 return tags 567 568 569 def latest_tag(tags, dev): 570 """Find the newest supported tag, only considering dev tags if dev is set""" 571 best = None 572 bestkey = None 573 # Iterate in sorted order, so that the result doesn't depend on 574 # the order in which the remote lists its refs. 575 for tag in sorted(tags): 576 if not dev and tag_re_dev.fullmatch(tag): 577 continue 578 key = tag_sortkey(tag) 579 if key is None: 580 continue 581 if bestkey is None or key > bestkey: 582 best = tag 583 bestkey = key 584 return best 585 586 587 def upgrade(cfg): 588 """Upgrade tag files in buildconfig to the latest upstream tag""" 589 names = cfg.components 590 if not names: 591 names = sorted(p.stem for p in Path("buildconfig").glob("*.tag")) 592 # Multiple components can share a repo, only ask each remote once. 593 remote_tags = {} 594 upgraded = [] 595 for name in names: 596 tag_file = Path("buildconfig") / f"{name}.tag" 597 url_file = Path("buildconfig") / f"{name}.giturl" 598 if not url_file.exists(): 599 print(f"[?] {name} has no giturl, skipping", file=sys.stderr) 600 continue 601 giturl = url_file.read_text().strip() 602 if giturl not in remote_tags: 603 remote_tags[giturl] = list_remote_tags(giturl) 604 latest = latest_tag(remote_tags[giturl], cfg.dev) 605 if latest is None: 606 print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr) 607 continue 608 curr = None 609 if tag_file.exists(): 610 curr = tag_file.read_text().strip() 611 currkey = tag_sortkey(curr) 612 if currkey is None: 613 print( 614 f"[?] {name} tag {curr} has unsupported syntax, skipping", 615 file=sys.stderr, 616 ) 617 continue 618 latestkey = tag_sortkey(latest) 619 if currkey > latestkey: 620 # Happens when the tag file pins a dev version but only 621 # production tags are considered. 622 print(f" {name} {curr} (newer than latest {latest})") 623 continue 624 if currkey == latestkey: 625 print(f" {name} {curr} (up to date)") 626 continue 627 print(f"[!] {name} {curr or '(none)'} -> {latest}") 628 upgraded.append(name) 629 if not cfg.dry: 630 tag_file.write_text(latest + "\n") 631 if not upgraded: 632 print("nothing to upgrade") 633 elif cfg.dry: 634 print("would upgrade:", " ".join(upgraded)) 635 else: 636 print("upgraded:", " ".join(upgraded)) 637 638 639 def main(): 640 parser = argparse.ArgumentParser( 641 prog="taler-pkg", description="Taler Packaging Helper" 642 ) 643 644 subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND") 645 646 # subcommand build 647 648 parser_build = subparsers.add_parser("build", help="Build packages for distro.") 649 parser_build.set_defaults(func=build) 650 parser_build.add_argument("distro") 651 # Keep for backwards compat 652 parser_build.add_argument( 653 "--no-transitive", 654 help="Do not build transitive deps of changed components (default)", 655 action="store_true", 656 dest="transitive", 657 default=None, 658 ) 659 parser_build.add_argument( 660 "--transitive", 661 help="Build transitive deps of changed components", 662 action="store_false", 663 dest="transitive", 664 default=None, 665 ) 666 parser_build.add_argument( 667 "--arch", 668 help="Architecture(s) to build for", 669 action="store", 670 dest="arch", 671 default=None, 672 ) 673 parser_build.add_argument( 674 "--dry", help="Dry run", action="store_true", default=False 675 ) 676 677 parser_test = subparsers.add_parser("test", help="Test packages for distro.") 678 parser_test.set_defaults(func=test) 679 parser_test.add_argument("distro") 680 parser_test.add_argument( 681 "--arch", 682 help="Architecture(s) to test packages for", 683 action="store", 684 dest="arch", 685 default=None, 686 ) 687 688 # subcommand show-latest 689 690 parser_show_latest = subparsers.add_parser( 691 "show-latest", help="Show latest version of packages." 692 ) 693 parser_show_latest.set_defaults(func=print_latest) 694 695 # subcommand upgrade 696 697 parser_upgrade = subparsers.add_parser( 698 "upgrade", help="Upgrade component tags to the latest upstream version." 699 ) 700 parser_upgrade.set_defaults(func=upgrade) 701 parser_upgrade.add_argument( 702 "components", 703 nargs="*", 704 help="Components to upgrade (default: all components in buildconfig)", 705 ) 706 parser_upgrade.add_argument( 707 "--dev", 708 help="Also consider dev tags, not just production tags", 709 action="store_true", 710 default=False, 711 ) 712 parser_upgrade.add_argument( 713 "--dry", help="Dry run", action="store_true", default=False 714 ) 715 716 # subcommand show-order 717 718 parser_show_order = subparsers.add_parser("show-order", help="Show build order.") 719 parser_show_order.set_defaults(func=show_order) 720 parser_show_order.add_argument("roots", nargs="+") 721 722 # subcommand show-published 723 parser_show_published = subparsers.add_parser( 724 "show-published", help="Show published packages on deb.taler.net" 725 ) 726 parser_show_published.add_argument("distro") 727 parser_show_published.set_defaults(func=show_published) 728 729 # subcommand publish 730 731 parser_publish = subparsers.add_parser("publish", help="Publish to deb.taler.net") 732 parser_publish.add_argument( 733 "--dry", help="Dry run", action="store_true", default=False 734 ) 735 parser_publish.add_argument("distro") 736 parser_publish.set_defaults(func=publish) 737 738 parser_promote = subparsers.add_parser("promote", help="Promote testing to stable") 739 parser_promote.add_argument( 740 "--dry", help="Dry run (show pulls)", action="store_true", default=False 741 ) 742 parser_promote.add_argument("distro") 743 parser_promote.set_defaults(func=promote) 744 745 args = parser.parse_args() 746 747 if "func" not in args: 748 parser.print_help() 749 else: 750 args.func(args) 751 752 753 if __name__ == "__main__": 754 main()