buildlib.py (9741B)
1 #!/usr/bin/env python3 2 3 # This file is in the public domain. 4 5 import json 6 import os 7 import shutil 8 import subprocess 9 import sys 10 from email.utils import formatdate 11 from pathlib import Path 12 13 from package_config import ConfigError, load_config 14 15 PKGDIR = Path("/pkgdir") 16 17 18 def run_cmd(cmd, shell=False, cwd=None, env=None): 19 """Run a command and stop the build if it fails.""" 20 command_env = os.environ.copy() 21 if env: 22 command_env.update(env) 23 sys.stdout.flush() 24 subprocess.check_call(cmd, shell=shell, cwd=cwd, env=command_env) 25 26 27 def get_output(cmd, shell=False, cwd=None): 28 return subprocess.check_output(cmd, shell=shell, cwd=cwd, text=True).strip() 29 30 31 def remove_stale_packages(pkgdir): 32 """Remove package artifacts not referenced by a current-build manifest.""" 33 pkgdir = Path(pkgdir) 34 current_packages = set() 35 for manifest in pkgdir.glob("*.built.current"): 36 current_packages.update(manifest.read_text().split()) 37 38 for artifact in sorted(pkgdir.iterdir()): 39 if artifact.suffix not in (".deb", ".ddeb"): 40 continue 41 if artifact.name in current_packages: 42 continue 43 print(f"Removing stale local package {artifact.name}") 44 artifact.unlink() 45 46 47 def get_tag_debver(tag): 48 """Get a Debian version string from a supported Git tag.""" 49 if tag.startswith("v"): 50 devsuff = "-dev." 51 position = tag.find(devsuff) 52 if position < 0: 53 return tag[1:] 54 return tag[1:position] + "~dev" + tag[position + len(devsuff) :] 55 if tag.startswith("deb-v"): 56 tag = tag[5:] 57 if "-" in tag: 58 version, revision = tag.split("-", 1) 59 return version + "-" + revision 60 return tag 61 raise ValueError(f"unexpected tag format: {tag}") 62 63 64 def make_codename_version(deb_version, build_codename): 65 if "-" in deb_version: 66 return f"{deb_version}+{build_codename}" 67 return f"{deb_version}-0+{build_codename}" 68 69 70 def scan_local_repository(): 71 os.chdir(PKGDIR) 72 packages_index = PKGDIR / "Packages.xz" 73 run_cmd(f"dpkg-scanpackages . | xz - > {packages_index}", shell=True) 74 with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as source: 75 source.write(f"deb [trusted=yes] file:{PKGDIR} ./\n") 76 run_cmd(["apt-get", "update"]) 77 78 79 def _validate_group(config, package_names, expected_builder): 80 if not package_names: 81 raise ConfigError("no packages specified") 82 if len(set(package_names)) != len(package_names): 83 raise ConfigError("a package was specified more than once") 84 85 packages = [] 86 keys = set() 87 for name in package_names: 88 try: 89 package = config.packages[name] 90 except KeyError as exc: 91 raise ConfigError(f"unknown package {name!r}") from exc 92 repository = config.repository_for(package) 93 builder = config.builder_for(package) 94 keys.add((repository.url, package.tag, builder)) 95 packages.append(package) 96 if len(keys) != 1: 97 raise ConfigError("grouped packages must have the same repository, tag, and builder") 98 repository_url, tag, builder = keys.pop() 99 if builder != expected_builder: 100 raise ConfigError(f"expected builder {expected_builder!r}, got {builder!r}") 101 if expected_builder == "generic" and len(packages) != 1: 102 raise ConfigError("the generic builder accepts exactly one package") 103 return packages, repository_url, tag 104 105 106 def _install_build_dependencies(source_dir, package_paths): 107 tool = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" 108 controls = [str(package_path / "debian" / "control") for package_path in package_paths] 109 run_cmd( 110 ["mk-build-deps", "--install", f"--tool={tool}", *controls], 111 cwd=source_dir, 112 ) 113 114 115 def _prepare_pnpm_workspace(source_dir, package_paths): 116 filters = [] 117 for package_path in package_paths: 118 package_json = package_path / "package.json" 119 with package_json.open(encoding="utf-8") as package_file: 120 workspace_name = json.load(package_file).get("name") 121 if not isinstance(workspace_name, str) or not workspace_name: 122 raise ConfigError(f"{package_json} has no package name") 123 filters.extend(["--filter", f"{workspace_name}..."]) 124 125 run_cmd(["pnpm", "install", "--frozen-lockfile", *filters], cwd=source_dir) 126 run_cmd(["pnpm", "run", *filters, "build"], cwd=source_dir) 127 128 129 def _write_changelog(package_path, package_name, version): 130 debian_date = formatdate(localtime=True) 131 changelog = f"""\ 132 {package_name} ({version}) unstable; urgency=low 133 134 * Release {version}. 135 136 -- Taler Packaging Team <deb@taler.net> {debian_date} 137 """ 138 (package_path / "debian" / "changelog").write_text(changelog) 139 140 141 def _package_artifacts(output_dir): 142 return { 143 artifact.resolve() 144 for pattern in ("*.deb", "*.ddeb") 145 for artifact in output_dir.glob(pattern) 146 } 147 148 149 def _check_installed_binaries(deb_files): 150 for deb in deb_files: 151 contents = get_output(["dpkg", "--contents", str(deb)]) 152 for line in contents.splitlines(): 153 parts = line.split() 154 if len(parts) < 6: 155 raise RuntimeError(f"failed to read package contents from {deb}") 156 filename = parts[5] 157 if "bin" not in filename: 158 continue 159 if filename.startswith("./"): 160 filename = filename[2:] 161 if not filename.startswith("/"): 162 filename = "/" + filename 163 file_info = get_output(["file", filename]) 164 if "ELF" in file_info and "executable" in file_info: 165 print(f"checking {filename}") 166 try: 167 run_cmd(["ldd", filename]) 168 except subprocess.CalledProcessError as exc: 169 raise RuntimeError( 170 f"installed binary {filename} has a linker issue" 171 ) from exc 172 173 174 def _build_package(package, package_path, tag, codename, arch, prebuilt): 175 version = make_codename_version(get_tag_debver(tag), codename) 176 print(f"Building {package.name} as version {version}", file=sys.stderr) 177 _write_changelog(package_path, package.name, version) 178 179 output_dir = package_path.parent 180 before = _package_artifacts(output_dir) 181 environment = {"DEB_BUILD_MAINT_OPTIONS": "debug"} 182 if prebuilt: 183 environment["TALER_PACKAGING_PREBUILT"] = "1" 184 185 debug_repository = package_path / "debian" / ".debhelper" 186 debug_repository.mkdir(parents=True, exist_ok=True) 187 (debug_repository / "debian-symbols-pool").touch() 188 environment["DEB_DBG_SYMBOLS_REPO"] = "debian/.debhelper/" 189 190 run_cmd( 191 ["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"], 192 cwd=package_path, 193 env=environment, 194 ) 195 artifacts = _package_artifacts(output_dir) - before 196 deb_files = sorted(path for path in artifacts if path.suffix == ".deb") 197 if not deb_files: 198 raise RuntimeError(f"{package.name} did not produce a Debian package") 199 200 print(f"Installing built packages from {output_dir}", file=sys.stderr) 201 run_cmd(["apt", "install", "-y", *map(str, deb_files)]) 202 _check_installed_binaries(deb_files) 203 204 for artifact in artifacts: 205 shutil.copy(artifact, PKGDIR) 206 manifest = PKGDIR / f"{package.name}@{arch}.built.current" 207 manifest.write_text("".join(f"{artifact.name}\n" for artifact in sorted(artifacts))) 208 (PKGDIR / f"{package.name}@{arch}.built.tag").write_text(tag + "\n") 209 remove_stale_packages(PKGDIR) 210 scan_local_repository() 211 212 213 def build_packages(codename, arch, package_names, expected_builder): 214 if "LD_LIBRARY_PATH" in os.environ: 215 del os.environ["LD_LIBRARY_PATH"] 216 217 config = load_config("/packages.toml") 218 packages, repository_url, tag = _validate_group( 219 config, package_names, expected_builder 220 ) 221 print( 222 f"Building {' '.join(package_names)} with {expected_builder} build logic", 223 file=sys.stderr, 224 ) 225 226 scan_local_repository() 227 source_dir = Path("/build/source") 228 source_dir.parent.mkdir(parents=True, exist_ok=True) 229 run_cmd(["git", "config", "--global", "advice.detachedHead", "false"]) 230 run_cmd( 231 [ 232 "git", 233 "clone", 234 "--depth=1", 235 f"--branch={tag}", 236 repository_url, 237 str(source_dir), 238 ] 239 ) 240 run_cmd(["./bootstrap"], cwd=source_dir) 241 242 package_paths = [source_dir / package.debian_path for package in packages] 243 for package, package_path in zip(packages, package_paths): 244 control = package_path / "debian" / "control" 245 if not control.is_file(): 246 raise ConfigError( 247 f"{package.name} has no debian/control at tag {tag}: {control}" 248 ) 249 (package_path / ".version").write_text(get_tag_debver(tag) + "\n") 250 251 _install_build_dependencies(source_dir, package_paths) 252 if expected_builder == "pnpm-workspace": 253 _prepare_pnpm_workspace(source_dir, package_paths) 254 255 for package, package_path in zip(packages, package_paths): 256 _build_package( 257 package, 258 package_path, 259 tag, 260 codename, 261 arch, 262 prebuilt=expected_builder == "pnpm-workspace", 263 ) 264 265 266 def main(expected_builder): 267 if len(sys.argv) < 4: 268 print( 269 f"Usage: {Path(sys.argv[0]).name} <CODENAME> <ARCH> <PACKAGE>...", 270 file=sys.stderr, 271 ) 272 sys.exit(1) 273 try: 274 build_packages(sys.argv[1], sys.argv[2], sys.argv[3:], expected_builder) 275 except ConfigError as exc: 276 print(f"configuration error: {exc}", file=sys.stderr) 277 sys.exit(1)