taler-pkg (18911B)
1 #!/usr/bin/env python3 2 3 # Copyright (c) 2024 Taler Systems SA 4 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 5 # SPDX-License-Identifier: GPL-3.0-or-later 6 7 import argparse 8 import subprocess 9 import platform 10 import os 11 import sys 12 from pathlib import Path 13 14 # Make local util package available 15 file = Path(__file__).resolve() 16 parent, root = file.parent, file.parents[1] 17 sys.path.append(str(root)) 18 19 from util import vercomp 20 21 mydir = os.path.dirname(os.path.realpath(__file__)) 22 23 archs = ["arm64", "amd64"] 24 host = "taler.net" 25 native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" 26 27 components = [ 28 "gnunet", 29 "libeufin", 30 "donau", 31 "challenger", 32 "taler-exchange", 33 "taler-harness", 34 "taler-merchant", 35 "taler-rust", 36 "robocop", 37 "taler-wallet-cli", 38 #"depolymerization", 39 # These two packages don't have good debs yet, 40 # Debian complains "No section given for ..., skipping. 41 # "taler-directory", 42 # "taler-mailbox", 43 # We don't publish packages for these yet 44 # "taler-mdb", 45 # "taler-merchant-demos", 46 # "anastasis", 47 # "anastasis-gtk", 48 # Currently not used anywhere 49 # "sync", 50 ] 51 52 deps = { 53 "taler-exchange": ["gnunet"], 54 "anastasis": ["gnunet", "taler-merchant"], 55 "anastasis-gtk": ["anastasis"], 56 "taler-merchant": ["gnunet", "taler-exchange"], 57 "donau": ["gnunet", "taler-exchange"], 58 # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"], 59 "sync": ["taler-merchant", "taler-exchange", "gnunet"], 60 } 61 62 # Compute reverse dependencies 63 rdeps = {} 64 for n1, d in deps.items(): 65 for n2 in d: 66 rd = rdeps.setdefault(n2, []) 67 if n1 not in rd: 68 rd.append(n1) 69 70 71 def buildsort(roots): 72 """Toposort transitive closure of roots based on deps""" 73 out = [] 74 stack = list(roots) 75 pmark = set() 76 tmark = set() 77 while len(stack): 78 node = stack[-1] 79 if node in pmark: 80 stack.pop() 81 tmark.discard(node) 82 continue 83 done = True 84 for dep in deps.get(node, []): 85 if dep not in pmark: 86 if dep in tmark: 87 raise Exception("cycle") 88 stack.append(dep) 89 tmark.add(node) 90 done = False 91 if done: 92 pmark.add(node) 93 out.append(node) 94 stack.pop() 95 tmark.discard(node) 96 return out 97 98 99 def propagate_outdated(outdated): 100 """Propagate outdatedness to dependees""" 101 closure = set() 102 q = list(outdated) 103 while len(q): 104 n = q.pop() 105 closure.add(n) 106 for r in rdeps.get(n, []): 107 if r not in closure: 108 closure.add(r) 109 q.append(r) 110 return closure 111 112 113 def find_outdated(pkgdir, arch, roots): 114 """Find outdated components based on tag files""" 115 outdated = set() 116 for component in roots: 117 ver_requested = open(f"buildconfig/{component}.tag").read().strip() 118 built_tag_file = pkgdir / f"{component}@{arch}.built.tag" 119 ver_built = None 120 if built_tag_file.exists(): 121 ver_built = open(built_tag_file).read().strip() 122 if ver_built != ver_requested: 123 outdated.add(component) 124 print(component, ver_built, "->", ver_requested) 125 return outdated 126 127 128 def build(cfg): 129 transitive = False 130 if cfg.transitive: 131 transitive = True 132 distro = cfg.distro 133 vendor, codename = distro.split("-", 1) 134 print("building", distro) 135 dockerfile = f"distros/{distro}.Dockerfile" 136 image_tag = f"localhost/taler-packaging-{distro}:latest" 137 pkgdir = Path(f"packages/{distro}").absolute() 138 cachedir = Path("cache").absolute() 139 cachedir.mkdir(exist_ok=True) 140 (cachedir / "cargo-git").mkdir(exist_ok=True) 141 (cachedir / "cargo-registry").mkdir(exist_ok=True) 142 (cachedir / "cargo-build").mkdir(exist_ok=True) 143 (cachedir / "gradle").mkdir(exist_ok=True) 144 (cachedir / "pnpm").mkdir(exist_ok=True) 145 (cachedir / distro / "apt-archives").mkdir(parents=True, exist_ok=True) 146 (cachedir / distro / "apt-lists").mkdir(parents=True, exist_ok=True) 147 148 if cfg.arch is None: 149 arch_list = [native_arch] 150 else: 151 arch_list = cfg.arch.split(",") 152 153 if not cfg.dry: 154 for arch in arch_list: 155 subprocess.run( 156 [ 157 "podman", 158 "build", 159 "--arch", 160 arch, 161 "-v", 162 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 163 "-v", 164 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 165 "-t", 166 image_tag, 167 "-f", 168 dockerfile, 169 ], 170 check=True, 171 ) 172 173 # Sort components by their dependencies 174 buildorder = buildsort(components) 175 print("build order:", buildorder) 176 177 for arch in arch_list: 178 outdated = find_outdated(pkgdir, arch, buildorder) 179 180 # Propagate outdatedness to dependees 181 closure = propagate_outdated(outdated) 182 183 print("outdated closure", closure) 184 185 for component in buildorder: 186 if transitive: 187 if component not in closure: 188 continue 189 else: 190 if component not in outdated: 191 continue 192 print("building", component) 193 pkgdir.mkdir(parents=True, exist_ok=True) 194 cmd = [ 195 "podman", 196 "run", 197 "-it", 198 "--arch", 199 arch, 200 "--entrypoint=/bin/python3", 201 "--security-opt", 202 "label=disable", 203 "--mount", 204 f"type=bind,source={cachedir}/gradle,target=/root/.gradle/caches", 205 "--mount", 206 f"type=bind,source={cachedir}/pnpm,target=/root/.local/share/pnpm/store", 207 "--mount", 208 f"type=bind,source={cachedir}/cargo-registry,target=/root/.cargo/registry", 209 "--mount", 210 f"type=bind,source={cachedir}/cargo-git,target=/root/.cargo/git", 211 "--mount", 212 f"type=bind,source={cachedir}/cargo-build,target=/root/.cargo-build", 213 "--env", 214 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 215 "--mount", 216 f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared", 217 "--mount", 218 f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared", 219 "--mount", 220 f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", 221 "--mount", 222 f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly", 223 "--mount", 224 f"type=bind,source={pkgdir},target=/pkgdir", 225 image_tag, 226 "/buildscripts/generic", 227 component, 228 codename, 229 arch, 230 ] 231 if not cfg.dry: 232 subprocess.run( 233 cmd, 234 check=True, 235 ) 236 237 238 def show_order(cfg): 239 buildorder = buildsort(list(cfg.roots)) 240 print("build order:", buildorder) 241 242 243 244 245 def promote(cfg): 246 dry = cfg.dry 247 distro = cfg.distro 248 vendor, codename = distro.split("-", 1) 249 listfmt = "${package}_${version}_${architecture}.${$type}\n" 250 if dry: 251 subprocess.run( 252 [ 253 "ssh", 254 f"taler-packaging@{host}", 255 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}", 256 ], 257 check=True, 258 ) 259 else: 260 subprocess.run( 261 [ 262 "ssh", 263 "-t", 264 f"taler-packaging@{host}", 265 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}", 266 ], 267 check=True, 268 ) 269 # Always export! 270 # Reprepro is weird, listed packages might actually not show 271 # up in the index yet. 272 subprocess.run( 273 [ 274 "ssh", 275 "-t", 276 f"taler-packaging@{host}", 277 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 278 ], 279 check=True, 280 ) 281 282 def show_published(cfg): 283 distro = cfg.distro 284 vendor, codename = distro.split("-", 1) 285 listfmt = "${package}_${version}_${architecture}.${$type}\n" 286 subprocess.run( 287 [ 288 "ssh", 289 f"taler-packaging@{host}", 290 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 291 ], 292 check=True, 293 ) 294 295 296 def test(cfg): 297 target = cfg.distro 298 vendor, codename, *rest = target.split("-") 299 distro = f"{vendor}-{codename}" 300 image_tag = f"localhost/taler-packaging-{distro}:latest" 301 dockerfile = f"distros/{distro}.Dockerfile" 302 cachedir = Path(f"cache").absolute() 303 print("building base image") 304 subprocess.run( 305 [ 306 "podman", 307 "build", 308 "-v", 309 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 310 "-v", 311 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 312 "-t", 313 image_tag, 314 "-f", 315 dockerfile, 316 ], 317 check=True, 318 ) 319 print("running test") 320 cmd = [ 321 "podman", 322 "run", 323 "-it", 324 "--entrypoint=/bin/bash", 325 "--security-opt", 326 "label=disable", 327 "--mount", 328 f"type=bind,source={mydir}/testing,target=/testing,readonly", 329 image_tag, 330 f"/testing/test-{target}", 331 ] 332 subprocess.run( 333 cmd, 334 check=True, 335 ) 336 337 def publish(cfg): 338 distro = cfg.distro 339 if distro.endswith("-testing"): 340 print("Files are automatically published to testing", file=sys.stderr) 341 sys.exit(1) 342 vendor, codename = distro.split("-", 1) 343 # List of .deb and .ddeb files. 344 debs = [] 345 listfmt = "${package}_${version}_${architecture}.${$type}\n" 346 server_debs_str = subprocess.check_output( 347 [ 348 "ssh", 349 f"taler-packaging@{host}", 350 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 351 ], 352 encoding="utf-8", 353 ) 354 server_debs = server_debs_str.split() 355 for component in components: 356 current = [] 357 for arch in archs + ["all"]: 358 cf = Path(f"./packages/{distro}/{component}@{arch}.built.current") 359 if not cf.exists(): 360 print(f"component {component}@{arch} has no current packages") 361 continue 362 with open(cf) as f: 363 current = current + f.read().split() 364 print("current", current) 365 for deb in current: 366 if deb.endswith(".deb"): 367 pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_") 368 elif deb.endswith(".ddeb"): 369 pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_") 370 else: 371 raise Error(f"invalid deb filename: {deb}") 372 fresh = True 373 server_deb = None 374 # If the server has the same or a later version, 375 # the local version isn't fresh. 376 for srvdeb in server_debs: 377 pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_") 378 if pkg1 != pkg2 or arch1 != arch2: 379 continue 380 if vercomp.compare_versions(ver1, ver2) <= 0: 381 fresh = False 382 server_deb = srvdeb 383 break 384 if fresh: 385 debs.append(deb) 386 else: 387 print("package", deb, "not fresh, server has", server_deb) 388 if len(debs) == 0: 389 print("nothing to upload") 390 else: 391 print("uploading debs", debs) 392 if cfg.dry: 393 return 394 debs = [Path(f"./packages/{distro}/") / x for x in debs] 395 subprocess.run( 396 [ 397 "ssh", 398 f"taler-packaging@{host}", 399 f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb", 400 ], 401 check=True, 402 ) 403 subprocess.run( 404 ["scp", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True 405 ) 406 ret = subprocess.run( 407 [ 408 "ssh", 409 "-t", 410 f"taler-packaging@{host}", 411 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb", 412 ], 413 ) 414 if ret.returncode != 0: 415 # Usually not critical if it fails. 416 print( 417 "Including ddebs failed. This can happen when including packages that have been included previously" 418 ) 419 # Almost the same, but with ddebs. 420 # We explicitly need to tell reprepro 421 # to ignore the extension, because it does not 422 # deal well with ddebs out of the box. 423 ret = subprocess.run( 424 [ 425 "ssh", 426 "-t", 427 f"taler-packaging@{host}", 428 f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb", 429 ], 430 ) 431 if ret.returncode != 0: 432 # Usually not critical if it fails. 433 print( 434 "Including ddebs failed. This can happen when including packages that have been included previously" 435 ) 436 # Always export! 437 # Reprepro is weird, listed packages might actually not show 438 # up in the index yet. 439 subprocess.run( 440 [ 441 "ssh", 442 "-t", 443 f"taler-packaging@{host}", 444 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 445 ], 446 check=True, 447 ) 448 449 450 def get_remote_version(url): 451 """Get the latest stable tag from the git repo""" 452 # Construct the git command 453 # We use -c versionsort.suffix=- to ensure correct semantic version sorting 454 cmd = [ 455 "git", 456 "-c", 457 "versionsort.suffix=-", 458 "ls-remote", 459 "--exit-code", 460 "--refs", 461 "--sort=version:refname", 462 "--tags", 463 url, 464 "*.*.*", 465 ] 466 467 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 468 469 # Parse the output 470 # Output format is usually: <hash>\trefs/tags/<tagname> 471 lines = result.stdout.strip().split("\n") 472 473 valid_tags = [] 474 475 for line in lines: 476 parts = line.split() 477 if len(parts) < 2: 478 continue 479 480 # refs/tags/v1.0.0 -> v1.0.0 481 ref_path = parts[1] 482 tag = ref_path.split("/")[-1] 483 484 # Exclude pre-release semver versions 485 if tag.startswith("v") and "-" in tag: 486 continue 487 488 valid_tags.append(tag) 489 490 if valid_tags: 491 return valid_tags[-1] 492 return "(none)" 493 494 495 def check_version(name, url): 496 """ 497 Compares local buildconfig version with remote git version. 498 """ 499 ver = get_remote_version(url) 500 config_path = os.path.join("buildconfig", f"{name}.tag") 501 with open(config_path, "r") as f: 502 curr = f.read().strip() 503 prefix = "[!] " if curr != ver else "" 504 print(f"{prefix}{name} curr: {curr} latest: {ver}") 505 506 507 def print_latest(cfg): 508 """Print latest upstream tag for each component""" 509 for name in components: 510 config_path = os.path.join("buildconfig", f"{name}.giturl") 511 with open(config_path, "r") as f: 512 giturl = f.read().strip() 513 check_version(name, giturl) 514 515 516 def main(): 517 parser = argparse.ArgumentParser( 518 prog="taler-pkg", description="Taler Packaging Helper" 519 ) 520 521 subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND") 522 523 # subcommand build 524 525 parser_build = subparsers.add_parser("build", help="Build packages for distro.") 526 parser_build.set_defaults(func=build) 527 parser_build.add_argument("distro") 528 # Keep for backwards compat 529 parser_build.add_argument( 530 "--no-transitive", 531 help="Do not build transitive deps of changed components (default)", 532 action="store_true", 533 dest="transitive", 534 default=None, 535 ) 536 parser_build.add_argument( 537 "--transitive", 538 help="Build transitive deps of changed components", 539 action="store_false", 540 dest="transitive", 541 default=None, 542 ) 543 parser_build.add_argument( 544 "--arch", 545 help="Architecture(s) to build for", 546 action="store", 547 dest="arch", 548 default=None, 549 ) 550 parser_build.add_argument( 551 "--dry", help="Dry run", action="store_true", default=False 552 ) 553 554 parser_test = subparsers.add_parser("test", help="Test packages for distro.") 555 parser_test.set_defaults(func=test) 556 parser_test.add_argument("distro") 557 parser_test.add_argument( 558 "--arch", 559 help="Architecture(s) to test packages for", 560 action="store", 561 dest="arch", 562 default=None, 563 ) 564 565 # subcommand show-latest 566 567 parser_show_latest = subparsers.add_parser( 568 "show-latest", help="Show latest version of packages." 569 ) 570 parser_show_latest.set_defaults(func=print_latest) 571 572 # subcommand show-order 573 574 parser_show_order = subparsers.add_parser("show-order", help="Show build order.") 575 parser_show_order.set_defaults(func=show_order) 576 parser_show_order.add_argument("roots", nargs="+") 577 578 # subcommand show-published 579 parser_show_published = subparsers.add_parser( 580 "show-published", help="Show published packages on deb.taler.net" 581 ) 582 parser_show_published.add_argument("distro") 583 parser_show_published.set_defaults(func=show_published) 584 585 # subcommand publish 586 587 parser_publish = subparsers.add_parser("publish", help="Publish to deb.taler.net") 588 parser_publish.add_argument( 589 "--dry", help="Dry run", action="store_true", default=False 590 ) 591 parser_publish.add_argument("distro") 592 parser_publish.set_defaults(func=publish) 593 594 parser_promote = subparsers.add_parser("promote", help="Promote testing to stable") 595 parser_promote.add_argument( 596 "--dry", help="Dry run (show pulls)", action="store_true", default=False 597 ) 598 parser_promote.add_argument("distro") 599 parser_promote.set_defaults(func=promote) 600 601 args = parser.parse_args() 602 603 if "func" not in args: 604 parser.print_help() 605 else: 606 args.func(args) 607 608 609 if __name__ == "__main__": 610 main()