taler-pkg (18948B)
1 #!/usr/bin/env python3 2 3 # Copyright (c) 2024 Taler Systems SA 4 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) 5 # SPDX-License-Identifier: GPL-3.0-or-later 6 7 import argparse 8 import subprocess 9 import platform 10 import os 11 import sys 12 from pathlib import Path 13 14 # Make local util package available 15 file = Path(__file__).resolve() 16 parent, root = file.parent, file.parents[1] 17 sys.path.append(str(root)) 18 19 from util import vercomp 20 21 mydir = os.path.dirname(os.path.realpath(__file__)) 22 23 archs = ["arm64", "amd64"] 24 host = "taler.net" 25 native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" 26 27 components = [ 28 "gnunet", 29 "libeufin", 30 "donau", 31 "challenger", 32 "taler-exchange", 33 "taler-harness", 34 "taler-merchant", 35 "taler-rust", 36 "robocop", 37 "taler-wallet-cli", 38 "taler-merchant-webui", 39 #"depolymerization", 40 # These two packages don't have good debs yet, 41 # Debian complains "No section given for ..., skipping. 42 # "taler-directory", 43 # "taler-mailbox", 44 # We don't publish packages for these yet 45 # "taler-mdb", 46 # "taler-merchant-demos", 47 # "anastasis", 48 # "anastasis-gtk", 49 # Currently not used anywhere 50 # "sync", 51 ] 52 53 deps = { 54 "taler-exchange": ["gnunet"], 55 "anastasis": ["gnunet", "taler-merchant"], 56 "anastasis-gtk": ["anastasis"], 57 "taler-merchant": ["gnunet", "taler-exchange", "donau"], 58 "donau": ["gnunet", "taler-exchange"], 59 # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"], 60 "sync": ["taler-merchant", "taler-exchange", "gnunet"], 61 } 62 63 # Compute reverse dependencies 64 rdeps = {} 65 for n1, d in deps.items(): 66 for n2 in d: 67 rd = rdeps.setdefault(n2, []) 68 if n1 not in rd: 69 rd.append(n1) 70 71 72 def buildsort(roots): 73 """Toposort transitive closure of roots based on deps""" 74 out = [] 75 stack = list(roots) 76 pmark = set() 77 tmark = set() 78 while len(stack): 79 node = stack[-1] 80 if node in pmark: 81 stack.pop() 82 tmark.discard(node) 83 continue 84 done = True 85 for dep in deps.get(node, []): 86 if dep not in pmark: 87 if dep in tmark: 88 raise Exception("cycle") 89 stack.append(dep) 90 tmark.add(node) 91 done = False 92 if done: 93 pmark.add(node) 94 out.append(node) 95 stack.pop() 96 tmark.discard(node) 97 return out 98 99 100 def propagate_outdated(outdated): 101 """Propagate outdatedness to dependees""" 102 closure = set() 103 q = list(outdated) 104 while len(q): 105 n = q.pop() 106 closure.add(n) 107 for r in rdeps.get(n, []): 108 if r not in closure: 109 closure.add(r) 110 q.append(r) 111 return closure 112 113 114 def find_outdated(pkgdir, arch, roots): 115 """Find outdated components based on tag files""" 116 outdated = set() 117 for component in roots: 118 ver_requested = open(f"buildconfig/{component}.tag").read().strip() 119 built_tag_file = pkgdir / f"{component}@{arch}.built.tag" 120 ver_built = None 121 if built_tag_file.exists(): 122 ver_built = open(built_tag_file).read().strip() 123 if ver_built != ver_requested: 124 outdated.add(component) 125 print(component, ver_built, "->", ver_requested) 126 return outdated 127 128 129 def build(cfg): 130 transitive = False 131 if cfg.transitive: 132 transitive = True 133 distro = cfg.distro 134 vendor, codename = distro.split("-", 1) 135 print("building", distro) 136 dockerfile = f"distros/{distro}.Dockerfile" 137 image_tag = f"localhost/taler-packaging-{distro}:latest" 138 pkgdir = Path(f"packages/{distro}").absolute() 139 cachedir = Path("cache").absolute() 140 cachedir.mkdir(exist_ok=True) 141 (cachedir / "cargo-git").mkdir(exist_ok=True) 142 (cachedir / "cargo-registry").mkdir(exist_ok=True) 143 (cachedir / "cargo-build").mkdir(exist_ok=True) 144 (cachedir / "gradle").mkdir(exist_ok=True) 145 (cachedir / "pnpm").mkdir(exist_ok=True) 146 (cachedir / distro / "apt-archives").mkdir(parents=True, exist_ok=True) 147 (cachedir / distro / "apt-lists").mkdir(parents=True, exist_ok=True) 148 149 if cfg.arch is None: 150 arch_list = [native_arch] 151 else: 152 arch_list = cfg.arch.split(",") 153 154 if not cfg.dry: 155 for arch in arch_list: 156 subprocess.run( 157 [ 158 "podman", 159 "build", 160 "--arch", 161 arch, 162 "-v", 163 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 164 "-v", 165 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 166 "-t", 167 image_tag, 168 "-f", 169 dockerfile, 170 ], 171 check=True, 172 ) 173 174 # Sort components by their dependencies 175 buildorder = buildsort(components) 176 print("build order:", buildorder) 177 178 for arch in arch_list: 179 outdated = find_outdated(pkgdir, arch, buildorder) 180 181 # Propagate outdatedness to dependees 182 closure = propagate_outdated(outdated) 183 184 print("outdated closure", closure) 185 186 for component in buildorder: 187 if transitive: 188 if component not in closure: 189 continue 190 else: 191 if component not in outdated: 192 continue 193 print("building", component) 194 pkgdir.mkdir(parents=True, exist_ok=True) 195 cmd = [ 196 "podman", 197 "run", 198 "-it", 199 "--arch", 200 arch, 201 "--entrypoint=/bin/python3", 202 "--security-opt", 203 "label=disable", 204 "--mount", 205 f"type=bind,source={cachedir}/gradle,target=/root/.gradle/caches", 206 "--mount", 207 f"type=bind,source={cachedir}/pnpm,target=/root/.local/share/pnpm/store", 208 "--mount", 209 f"type=bind,source={cachedir}/cargo-registry,target=/root/.cargo/registry", 210 "--mount", 211 f"type=bind,source={cachedir}/cargo-git,target=/root/.cargo/git", 212 "--mount", 213 f"type=bind,source={cachedir}/cargo-build,target=/root/.cargo-build", 214 "--env", 215 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 216 "--mount", 217 f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared", 218 "--mount", 219 f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared", 220 "--mount", 221 f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", 222 "--mount", 223 f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly", 224 "--mount", 225 f"type=bind,source={pkgdir},target=/pkgdir", 226 image_tag, 227 "/buildscripts/generic", 228 component, 229 codename, 230 arch, 231 ] 232 if not cfg.dry: 233 subprocess.run( 234 cmd, 235 check=True, 236 ) 237 238 239 def show_order(cfg): 240 buildorder = buildsort(list(cfg.roots)) 241 print("build order:", buildorder) 242 243 244 245 246 def promote(cfg): 247 dry = cfg.dry 248 distro = cfg.distro 249 vendor, codename = distro.split("-", 1) 250 listfmt = "${package}_${version}_${architecture}.${$type}\n" 251 if dry: 252 subprocess.run( 253 [ 254 "ssh", 255 f"taler-packaging@{host}", 256 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ checkpull {codename}", 257 ], 258 check=True, 259 ) 260 else: 261 subprocess.run( 262 [ 263 "ssh", 264 "-t", 265 f"taler-packaging@{host}", 266 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ pull {codename}", 267 ], 268 check=True, 269 ) 270 # Always export! 271 # Reprepro is weird, listed packages might actually not show 272 # up in the index yet. 273 subprocess.run( 274 [ 275 "ssh", 276 "-t", 277 f"taler-packaging@{host}", 278 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 279 ], 280 check=True, 281 ) 282 283 def show_published(cfg): 284 distro = cfg.distro 285 vendor, codename = distro.split("-", 1) 286 listfmt = "${package}_${version}_${architecture}.${$type}\n" 287 subprocess.run( 288 [ 289 "ssh", 290 f"taler-packaging@{host}", 291 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 292 ], 293 check=True, 294 ) 295 296 297 def test(cfg): 298 target = cfg.distro 299 vendor, codename, *rest = target.split("-") 300 distro = f"{vendor}-{codename}" 301 image_tag = f"localhost/taler-packaging-{distro}:latest" 302 dockerfile = f"distros/{distro}.Dockerfile" 303 cachedir = Path(f"cache").absolute() 304 print("building base image") 305 subprocess.run( 306 [ 307 "podman", 308 "build", 309 "-v", 310 f"{cachedir}/{distro}/apt-archives:/var/cache/apt/archives:z", 311 "-v", 312 f"{cachedir}/{distro}/apt-lists:/var/lib/apt/lists:z", 313 "-t", 314 image_tag, 315 "-f", 316 dockerfile, 317 ], 318 check=True, 319 ) 320 print("running test") 321 cmd = [ 322 "podman", 323 "run", 324 "-it", 325 "--entrypoint=/bin/bash", 326 "--security-opt", 327 "label=disable", 328 "--mount", 329 f"type=bind,source={mydir}/testing,target=/testing,readonly", 330 image_tag, 331 f"/testing/test-{target}", 332 ] 333 subprocess.run( 334 cmd, 335 check=True, 336 ) 337 338 def publish(cfg): 339 distro = cfg.distro 340 if distro.endswith("-testing"): 341 print("Files are automatically published to testing", file=sys.stderr) 342 sys.exit(1) 343 vendor, codename = distro.split("-", 1) 344 # List of .deb and .ddeb files. 345 debs = [] 346 listfmt = "${package}_${version}_${architecture}.${$type}\n" 347 server_debs_str = subprocess.check_output( 348 [ 349 "ssh", 350 f"taler-packaging@{host}", 351 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ --list-format '{listfmt}' list {codename}", 352 ], 353 encoding="utf-8", 354 ) 355 server_debs = server_debs_str.split() 356 for component in components: 357 current = [] 358 for arch in archs + ["all"]: 359 cf = Path(f"./packages/{distro}/{component}@{arch}.built.current") 360 if not cf.exists(): 361 print(f"component {component}@{arch} has no current packages") 362 continue 363 with open(cf) as f: 364 current = current + f.read().split() 365 print("current", current) 366 for deb in current: 367 if deb.endswith(".deb"): 368 pkg1, ver1, arch1 = deb.removesuffix(".deb").split("_") 369 elif deb.endswith(".ddeb"): 370 pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_") 371 else: 372 raise Error(f"invalid deb filename: {deb}") 373 fresh = True 374 server_deb = None 375 # If the server has the same or a later version, 376 # the local version isn't fresh. 377 for srvdeb in server_debs: 378 pkg2, ver2, arch2 = srvdeb.removesuffix(".deb").split("_") 379 if pkg1 != pkg2 or arch1 != arch2: 380 continue 381 if vercomp.compare_versions(ver1, ver2) <= 0: 382 fresh = False 383 server_deb = srvdeb 384 break 385 if fresh: 386 debs.append(deb) 387 else: 388 print("package", deb, "not fresh, server has", server_deb) 389 if len(debs) == 0: 390 print("nothing to upload") 391 else: 392 print("uploading debs", debs) 393 if cfg.dry: 394 return 395 debs = [Path(f"./packages/{distro}/") / x for x in debs] 396 subprocess.run( 397 [ 398 "ssh", 399 f"taler-packaging@{host}", 400 f"rm -f '/home/taler-packaging/{distro}/'*.deb '/home/taler-packaging/{distro}/'*.ddeb", 401 ], 402 check=True, 403 ) 404 subprocess.run( 405 ["scp", "--", *debs, f"taler-packaging@{host}:{distro}/"], check=True 406 ) 407 ret = subprocess.run( 408 [ 409 "ssh", 410 "-t", 411 f"taler-packaging@{host}", 412 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.deb", 413 ], 414 ) 415 if ret.returncode != 0: 416 # Usually not critical if it fails. 417 print( 418 "Including ddebs failed. This can happen when including packages that have been included previously" 419 ) 420 # Almost the same, but with ddebs. 421 # We explicitly need to tell reprepro 422 # to ignore the extension, because it does not 423 # deal well with ddebs out of the box. 424 ret = subprocess.run( 425 [ 426 "ssh", 427 "-t", 428 f"taler-packaging@{host}", 429 f"reprepro --ignore=extension -b /home/taler-packaging/www/apt/{vendor}/ includedeb {codename}-testing ~/{vendor}-{codename}/*.ddeb", 430 ], 431 ) 432 if ret.returncode != 0: 433 # Usually not critical if it fails. 434 print( 435 "Including ddebs failed. This can happen when including packages that have been included previously" 436 ) 437 # Always export! 438 # Reprepro is weird, listed packages might actually not show 439 # up in the index yet. 440 subprocess.run( 441 [ 442 "ssh", 443 "-t", 444 f"taler-packaging@{host}", 445 f"reprepro -b /home/taler-packaging/www/apt/{vendor}/ export {codename}-testing", 446 ], 447 check=True, 448 ) 449 450 451 def get_remote_version(url): 452 """Get the latest stable tag from the git repo""" 453 # Construct the git command 454 # We use -c versionsort.suffix=- to ensure correct semantic version sorting 455 cmd = [ 456 "git", 457 "-c", 458 "versionsort.suffix=-", 459 "ls-remote", 460 "--exit-code", 461 "--refs", 462 "--sort=version:refname", 463 "--tags", 464 url, 465 "*.*.*", 466 ] 467 468 result = subprocess.run(cmd, capture_output=True, text=True, check=True) 469 470 # Parse the output 471 # Output format is usually: <hash>\trefs/tags/<tagname> 472 lines = result.stdout.strip().split("\n") 473 474 valid_tags = [] 475 476 for line in lines: 477 parts = line.split() 478 if len(parts) < 2: 479 continue 480 481 # refs/tags/v1.0.0 -> v1.0.0 482 ref_path = parts[1] 483 tag = ref_path.split("/")[-1] 484 485 # Exclude pre-release semver versions 486 if tag.startswith("v") and "-" in tag: 487 continue 488 489 valid_tags.append(tag) 490 491 if valid_tags: 492 return valid_tags[-1] 493 return "(none)" 494 495 496 def check_version(name, url): 497 """ 498 Compares local buildconfig version with remote git version. 499 """ 500 ver = get_remote_version(url) 501 config_path = os.path.join("buildconfig", f"{name}.tag") 502 with open(config_path, "r") as f: 503 curr = f.read().strip() 504 prefix = "[!] " if curr != ver else "" 505 print(f"{prefix}{name} curr: {curr} latest: {ver}") 506 507 508 def print_latest(cfg): 509 """Print latest upstream tag for each component""" 510 for name in components: 511 config_path = os.path.join("buildconfig", f"{name}.giturl") 512 with open(config_path, "r") as f: 513 giturl = f.read().strip() 514 check_version(name, giturl) 515 516 517 def main(): 518 parser = argparse.ArgumentParser( 519 prog="taler-pkg", description="Taler Packaging Helper" 520 ) 521 522 subparsers = parser.add_subparsers(help="Run a subcommand", metavar="SUBCOMMAND") 523 524 # subcommand build 525 526 parser_build = subparsers.add_parser("build", help="Build packages for distro.") 527 parser_build.set_defaults(func=build) 528 parser_build.add_argument("distro") 529 # Keep for backwards compat 530 parser_build.add_argument( 531 "--no-transitive", 532 help="Do not build transitive deps of changed components (default)", 533 action="store_true", 534 dest="transitive", 535 default=None, 536 ) 537 parser_build.add_argument( 538 "--transitive", 539 help="Build transitive deps of changed components", 540 action="store_false", 541 dest="transitive", 542 default=None, 543 ) 544 parser_build.add_argument( 545 "--arch", 546 help="Architecture(s) to build for", 547 action="store", 548 dest="arch", 549 default=None, 550 ) 551 parser_build.add_argument( 552 "--dry", help="Dry run", action="store_true", default=False 553 ) 554 555 parser_test = subparsers.add_parser("test", help="Test packages for distro.") 556 parser_test.set_defaults(func=test) 557 parser_test.add_argument("distro") 558 parser_test.add_argument( 559 "--arch", 560 help="Architecture(s) to test packages for", 561 action="store", 562 dest="arch", 563 default=None, 564 ) 565 566 # subcommand show-latest 567 568 parser_show_latest = subparsers.add_parser( 569 "show-latest", help="Show latest version of packages." 570 ) 571 parser_show_latest.set_defaults(func=print_latest) 572 573 # subcommand show-order 574 575 parser_show_order = subparsers.add_parser("show-order", help="Show build order.") 576 parser_show_order.set_defaults(func=show_order) 577 parser_show_order.add_argument("roots", nargs="+") 578 579 # subcommand show-published 580 parser_show_published = subparsers.add_parser( 581 "show-published", help="Show published packages on deb.taler.net" 582 ) 583 parser_show_published.add_argument("distro") 584 parser_show_published.set_defaults(func=show_published) 585 586 # subcommand publish 587 588 parser_publish = subparsers.add_parser("publish", help="Publish to deb.taler.net") 589 parser_publish.add_argument( 590 "--dry", help="Dry run", action="store_true", default=False 591 ) 592 parser_publish.add_argument("distro") 593 parser_publish.set_defaults(func=publish) 594 595 parser_promote = subparsers.add_parser("promote", help="Promote testing to stable") 596 parser_promote.add_argument( 597 "--dry", help="Dry run (show pulls)", action="store_true", default=False 598 ) 599 parser_promote.add_argument("distro") 600 parser_promote.set_defaults(func=promote) 601 602 args = parser.parse_args() 603 604 if "func" not in args: 605 parser.print_help() 606 else: 607 args.func(args) 608 609 610 if __name__ == "__main__": 611 main()