sandcastle_build_packages.py (14952B)
1 #!/usr/bin/env python3 2 # This file is in the public domain. 3 """Build outdated Sandcastle Debian packages in fresh containers.""" 4 5 from __future__ import annotations 6 7 import argparse 8 import hashlib 9 import json 10 import os 11 import shutil 12 import subprocess 13 import sys 14 import uuid 15 from collections.abc import Callable, Iterable, Mapping, Sequence 16 from pathlib import Path 17 18 try: 19 from .package_config import PackagingConfig, load_config 20 except ImportError: # Direct script execution inside the builder container. 21 from package_config import PackagingConfig, load_config 22 23 24 DISTRO = "debian-trixie" 25 BUILDER_IMAGE = "localhost/taler-sandcastle-package-builder:latest" 26 BuildRunner = Callable[[Sequence[str], Path], None] 27 28 29 def topological_order( 30 components: Sequence[str], dependencies: Mapping[str, Sequence[str]] 31 ) -> list[str]: 32 """Return components with every dependency before its consumers.""" 33 component_set = set(components) 34 if len(component_set) != len(components): 35 raise ValueError("component list contains duplicates") 36 order: list[str] = [] 37 permanent: set[str] = set() 38 temporary: set[str] = set() 39 40 def visit(component: str) -> None: 41 if component in permanent: 42 return 43 if component in temporary: 44 raise ValueError(f"dependency cycle involving {component}") 45 if component not in component_set: 46 raise ValueError(f"unknown or disabled package dependency: {component}") 47 temporary.add(component) 48 for dependency in dependencies.get(component, ()): 49 visit(dependency) 50 temporary.remove(component) 51 permanent.add(component) 52 order.append(component) 53 54 for component in components: 55 visit(component) 56 return order 57 58 59 def canonical_json(value: object) -> bytes: 60 return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() 61 62 63 def file_digest(path: Path) -> str: 64 digest = hashlib.sha256() 65 with path.open("rb") as source: 66 for chunk in iter(lambda: source.read(1024 * 1024), b""): 67 digest.update(chunk) 68 return digest.hexdigest() 69 70 71 def files_digest(paths: Iterable[Path]) -> str: 72 inputs = { 73 path.name: file_digest(path) 74 for path in sorted(paths) 75 if path.is_file() and not path.name.startswith("__pycache__") 76 } 77 return hashlib.sha256(canonical_json(inputs)).hexdigest() 78 79 80 def component_input_digest(config: PackagingConfig, component: str) -> str: 81 """Hash the resolved configuration that belongs to one package.""" 82 package = config.packages[component] 83 repository = config.repository_for(package) 84 inputs = { 85 "repository_url": repository.url, 86 "builder": config.builder_for(package), 87 "tag": package.tag, 88 "debian_path": package.debian_path, 89 "dependencies": package.dependencies, 90 "enabled": package.enabled, 91 } 92 return hashlib.sha256(canonical_json(inputs)).hexdigest() 93 94 95 def desired_build_keys( 96 order: Sequence[str], 97 dependencies: Mapping[str, Sequence[str]], 98 component_inputs: Mapping[str, str], 99 *, 100 distro: str, 101 architecture: str, 102 builder_image_id: str, 103 build_scripts_digest: str, 104 ) -> dict[str, str]: 105 """Compute recursive keys so dependency changes invalidate consumers.""" 106 keys: dict[str, str] = {} 107 for component in order: 108 payload = { 109 "schema": 2, 110 "component": component, 111 "component_input": component_inputs[component], 112 "distro": distro, 113 "architecture": architecture, 114 "builder_image_id": builder_image_id, 115 "build_scripts": build_scripts_digest, 116 "dependencies": { 117 dependency: keys[dependency] 118 for dependency in sorted(dependencies.get(component, ())) 119 }, 120 } 121 keys[component] = hashlib.sha256(canonical_json(payload)).hexdigest() 122 return keys 123 124 125 def component_group_key(config: PackagingConfig, component: str) -> tuple[str, ...]: 126 package = config.packages[component] 127 repository = config.repository_for(package) 128 builder = config.builder_for(package) 129 if builder == "generic": 130 return repository.url, package.tag, builder, component 131 return repository.url, package.tag, builder, "" 132 133 134 def group_components( 135 config: PackagingConfig, selected: Iterable[str], build_order: Sequence[str] 136 ) -> list[list[str]]: 137 """Group selected packages without violating package dependency order.""" 138 selected_set = set(selected) 139 groups: dict[tuple[str, ...], list[str]] = {} 140 component_to_group: dict[str, tuple[str, ...]] = {} 141 for component in build_order: 142 if component not in selected_set: 143 continue 144 key = component_group_key(config, component) 145 groups.setdefault(key, []).append(component) 146 component_to_group[component] = key 147 148 group_dependencies = {key: set() for key in groups} 149 for component, key in component_to_group.items(): 150 for dependency in config.packages[component].dependencies: 151 dependency_key = component_to_group.get(dependency) 152 if dependency_key is not None and dependency_key != key: 153 group_dependencies[key].add(dependency_key) 154 155 ordered: list[list[str]] = [] 156 permanent: set[tuple[str, ...]] = set() 157 temporary: set[tuple[str, ...]] = set() 158 159 def visit(key: tuple[str, ...]) -> None: 160 if key in permanent: 161 return 162 if key in temporary: 163 raise ValueError("build groups contain a dependency cycle") 164 temporary.add(key) 165 for dependency in sorted(group_dependencies[key]): 166 visit(dependency) 167 temporary.remove(key) 168 permanent.add(key) 169 ordered.append(groups[key]) 170 171 for key in groups: 172 visit(key) 173 return ordered 174 175 176 def state_path(state_dir: Path, component: str) -> Path: 177 return state_dir / f"{component}.key" 178 179 180 def package_files(artifact_dir: Path, component: str) -> list[Path]: 181 return sorted((artifact_dir / component).glob("*.deb")) 182 183 184 def component_is_current( 185 artifact_dir: Path, state_dir: Path, component: str, desired_key: str 186 ) -> bool: 187 try: 188 recorded_key = state_path(state_dir, component).read_text(encoding="utf-8").strip() 189 except FileNotFoundError: 190 return False 191 return recorded_key == desired_key and bool(package_files(artifact_dir, component)) 192 193 194 def write_state_atomically(path: Path, desired_key: str) -> None: 195 path.parent.mkdir(parents=True, exist_ok=True) 196 temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") 197 temporary.write_text(f"{desired_key}\n", encoding="utf-8") 198 os.replace(temporary, path) 199 200 201 def promote_artifacts(staging_dir: Path, artifact_dir: Path, component: str) -> None: 202 """Replace one package's artifacts while retaining the old set on failure.""" 203 destination = artifact_dir / component 204 artifact_dir.mkdir(parents=True, exist_ok=True) 205 backup = staging_dir.with_name(f".{component}.{uuid.uuid4().hex}.old") 206 if destination.exists(): 207 os.replace(destination, backup) 208 try: 209 os.replace(staging_dir, destination) 210 except BaseException: 211 if backup.exists(): 212 os.replace(backup, destination) 213 raise 214 if backup.exists(): 215 shutil.rmtree(backup) 216 217 218 def rebuild_outdated( 219 order: Sequence[str], 220 config: PackagingConfig, 221 desired_keys: Mapping[str, str], 222 artifact_dir: Path, 223 state_dir: Path, 224 staging_root: Path, 225 run_build: BuildRunner, 226 *, 227 force: bool = False, 228 ) -> list[str]: 229 """Build outdated packages in groups and promote only complete group outputs.""" 230 selected = [ 231 component 232 for component in order 233 if force 234 or not component_is_current( 235 artifact_dir, state_dir, component, desired_keys[component] 236 ) 237 ] 238 selected_set = set(selected) 239 for component in order: 240 status = "rebuilding" if component in selected_set else "up to date" 241 print(f"Package {component}: {status}") 242 243 rebuilt: list[str] = [] 244 for group in group_components(config, selected, order): 245 group_name = "+".join(group) 246 staging_dir = staging_root / f"{group_name}.{uuid.uuid4().hex}" 247 staging_dir.mkdir(parents=True, exist_ok=False) 248 for component in group: 249 (staging_dir / component).mkdir() 250 try: 251 run_build(group, staging_dir) 252 missing = [ 253 component 254 for component in group 255 if not sorted((staging_dir / component).glob("*.deb")) 256 ] 257 if missing: 258 raise RuntimeError( 259 "build produced no .deb packages for: " + ", ".join(missing) 260 ) 261 for component in group: 262 promote_artifacts(staging_dir / component, artifact_dir, component) 263 for component in group: 264 write_state_atomically( 265 state_path(state_dir, component), desired_keys[component] 266 ) 267 rebuilt.extend(group) 268 finally: 269 if staging_dir.exists(): 270 shutil.rmtree(staging_dir) 271 return rebuilt 272 273 274 def inspect_builder(image: str) -> tuple[str, str]: 275 result = subprocess.run( 276 ["podman", "image", "inspect", "--format", "{{.Id}} {{.Architecture}}", image], 277 check=True, 278 stdout=subprocess.PIPE, 279 text=True, 280 ) 281 fields = result.stdout.strip().split() 282 if len(fields) != 2: 283 raise RuntimeError(f"could not determine identity and architecture of {image}") 284 return fields[0], fields[1] 285 286 287 def ensure_directories(paths: Iterable[Path]) -> None: 288 for path in paths: 289 path.mkdir(parents=True, exist_ok=True) 290 291 292 CACHE_MOUNTS = ( 293 ("gradle", "/root/.gradle/caches"), 294 ("pnpm", "/root/.local/share/pnpm/store"), 295 ("cargo-registry", "/root/.cargo/registry"), 296 ("cargo-git", "/root/.cargo/git"), 297 ("cargo-build", "/root/.cargo-build"), 298 ("npm", "/root/.npm"), 299 ("go-build", "/root/.cache/go-build"), 300 ("go-mod", "/root/go/pkg/mod"), 301 ("pip", "/root/.cache/pip"), 302 ) 303 304 305 def podman_build_runner( 306 *, 307 root: Path, 308 image: str, 309 architecture: str, 310 distro: str, 311 codename: str, 312 artifact_dir: Path, 313 cache_dir: Path, 314 ) -> BuildRunner: 315 buildscripts_dir = root / "buildscripts" 316 config_path = root / "packages.toml" 317 mounts = tuple((cache_dir / source, target) for source, target in CACHE_MOUNTS) + ( 318 (cache_dir / distro / "apt-archives", "/var/cache/apt/archives"), 319 (cache_dir / distro / "apt-lists", "/var/lib/apt/lists"), 320 ) 321 322 def run_build(components: Sequence[str], staging_dir: Path) -> None: 323 command = [ 324 "podman", 325 "run", 326 "--rm", 327 "--arch", 328 architecture, 329 "--ulimit=nofile=2048:2048", 330 "--entrypoint=/bin/python3", 331 "--security-opt", 332 "label=disable", 333 ] 334 for source, target in mounts: 335 command.extend(["--mount", f"type=bind,source={source},target={target}"]) 336 command.extend( 337 [ 338 "--env", 339 "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", 340 "--mount", 341 f"type=bind,source={buildscripts_dir},target=/buildscripts,readonly", 342 "--mount", 343 f"type=bind,source={config_path},target=/packages.toml,readonly", 344 "--mount", 345 f"type=bind,source={artifact_dir},target=/pkgdir", 346 "--mount", 347 f"type=bind,source={staging_dir},target=/out", 348 image, 349 "/buildscripts/sandcastle-build-generic", 350 codename, 351 architecture, 352 *components, 353 ] 354 ) 355 subprocess.run(command, check=True) 356 357 return run_build 358 359 360 def parse_args(argv: Sequence[str]) -> argparse.Namespace: 361 parser = argparse.ArgumentParser(description=__doc__) 362 parser.add_argument("--builder-image", default=BUILDER_IMAGE) 363 parser.add_argument("--distro", default=DISTRO) 364 parser.add_argument("--force", action="store_true") 365 return parser.parse_args(argv) 366 367 368 def main(argv: Sequence[str] | None = None) -> int: 369 args = parse_args(sys.argv[1:] if argv is None else argv) 370 root = Path(__file__).resolve().parents[1] 371 config = load_config(root / "packages.toml") 372 artifact_dir = root / "packages" / args.distro 373 state_dir = root / "packages" / ".state" / args.distro 374 staging_root = root / "packages" / ".staging" / args.distro 375 cache_dir = root / "cache" 376 377 vendor, separator, codename = args.distro.partition("-") 378 if not separator or vendor != "debian" or not codename: 379 raise ValueError(f"unsupported distribution: {args.distro}") 380 builder_image_id, architecture = inspect_builder(args.builder_image) 381 artifact_dir /= architecture 382 state_dir /= architecture 383 staging_root /= architecture 384 385 cache_paths = [cache_dir / source for source, _target in CACHE_MOUNTS] 386 cache_paths.extend( 387 [ 388 cache_dir / args.distro / "apt-archives" / "partial", 389 cache_dir / args.distro / "apt-lists" / "partial", 390 artifact_dir, 391 state_dir, 392 staging_root, 393 ] 394 ) 395 ensure_directories(cache_paths) 396 397 components = config.enabled_packages() 398 dependencies = { 399 name: config.packages[name].dependencies 400 for name in components 401 if config.packages[name].dependencies 402 } 403 order = topological_order(components, dependencies) 404 component_inputs = { 405 component: component_input_digest(config, component) for component in order 406 } 407 keys = desired_build_keys( 408 order, 409 dependencies, 410 component_inputs, 411 distro=args.distro, 412 architecture=architecture, 413 builder_image_id=builder_image_id, 414 build_scripts_digest=files_digest((root / "buildscripts").glob("*")), 415 ) 416 runner = podman_build_runner( 417 root=root, 418 image=args.builder_image, 419 architecture=architecture, 420 distro=args.distro, 421 codename=codename, 422 artifact_dir=artifact_dir, 423 cache_dir=cache_dir, 424 ) 425 rebuilt = rebuild_outdated( 426 order, 427 config, 428 keys, 429 artifact_dir, 430 state_dir, 431 staging_root, 432 runner, 433 force=args.force, 434 ) 435 missing = [ 436 component 437 for component in order 438 if not component_is_current(artifact_dir, state_dir, component, keys[component]) 439 ] 440 if missing: 441 raise RuntimeError("packages are not current: " + ", ".join(missing)) 442 print("Rebuilt packages: " + ", ".join(rebuilt) if rebuilt else "All packages are up to date") 443 return 0 444 445 446 if __name__ == "__main__": 447 raise SystemExit(main())