test_sandcastle_build_packages.py (11386B)
1 # This file is in the public domain. 2 3 import runpy 4 import shutil 5 import subprocess 6 import tempfile 7 import unittest 8 from dataclasses import replace 9 from pathlib import Path 10 from unittest.mock import patch 11 12 from buildscripts.package_config import parse_config 13 from buildscripts.sandcastle_build_packages import ( 14 CACHE_MOUNTS, 15 component_input_digest, 16 component_is_current, 17 desired_build_keys, 18 group_components, 19 podman_build_runner, 20 rebuild_outdated, 21 topological_order, 22 ) 23 24 25 def packaging_config(*, workspace=True): 26 repositories = { 27 "generic": {"url": "https://example.test/generic.git"}, 28 "workspace": { 29 "url": "https://example.test/workspace.git", 30 "builder": "pnpm-workspace" if workspace else "generic", 31 }, 32 } 33 packages = { 34 "a": { 35 "repository": "generic", 36 "tag": "v1.0.0", 37 }, 38 "b": { 39 "repository": "workspace", 40 "tag": "v2.0.0", 41 "debian_path": "packages/b", 42 }, 43 "c": { 44 "repository": "workspace", 45 "tag": "v2.0.0", 46 "debian_path": "packages/c", 47 }, 48 } 49 return parse_config({"repositories": repositories, "packages": packages}) 50 51 52 class DebianVersionTests(unittest.TestCase): 53 @classmethod 54 def setUpClass(cls): 55 script = ( 56 Path(__file__).resolve().parents[1] 57 / "buildscripts" 58 / "sandcastle-build-generic" 59 ) 60 cls.generic = runpy.run_path(str(script)) 61 62 def test_development_tag_keeps_the_complete_sequence_number(self): 63 self.assertEqual("1.2.3~dev17", self.generic["get_tag_debver"]("v1.2.3-dev.17")) 64 65 def test_debian_revision_and_codename_are_preserved(self): 66 version = self.generic["get_tag_debver"]("deb-v1.2.3-4") 67 self.assertEqual( 68 "1.2.3-4+trixie", 69 self.generic["make_codename_version"](version, "trixie"), 70 ) 71 72 73 class DependencyKeyTests(unittest.TestCase): 74 dependencies = {"b": ("a",), "c": ("b",)} 75 components = ("c", "d", "b", "a") 76 77 def keys(self, inputs): 78 order = topological_order(self.components, self.dependencies) 79 return desired_build_keys( 80 order, 81 self.dependencies, 82 inputs, 83 distro="debian-trixie", 84 architecture="amd64", 85 builder_image_id="builder-1", 86 build_scripts_digest="scripts-1", 87 ) 88 89 def test_topological_order_puts_dependencies_first(self): 90 order = topological_order(self.components, self.dependencies) 91 self.assertLess(order.index("a"), order.index("b")) 92 self.assertLess(order.index("b"), order.index("c")) 93 self.assertEqual(set(self.components), set(order)) 94 95 def test_dependency_change_invalidates_reverse_dependencies(self): 96 before = self.keys({"a": "a1", "b": "b1", "c": "c1", "d": "d1"}) 97 after = self.keys({"a": "a2", "b": "b1", "c": "c1", "d": "d1"}) 98 changed = {name for name in before if before[name] != after[name]} 99 self.assertEqual({"a", "b", "c"}, changed) 100 101 def test_consumer_change_does_not_invalidate_dependency(self): 102 before = self.keys({"a": "a1", "b": "b1", "c": "c1", "d": "d1"}) 103 after = self.keys({"a": "a1", "b": "b2", "c": "c1", "d": "d1"}) 104 changed = {name for name in before if before[name] != after[name]} 105 self.assertEqual({"b", "c"}, changed) 106 107 108 class ComponentInputDigestTests(unittest.TestCase): 109 def test_unrelated_package_change_does_not_change_digest(self): 110 config = packaging_config() 111 before = component_input_digest(config, "a") 112 packages = dict(config.packages) 113 packages["b"] = replace(packages["b"], tag="v9.0.0") 114 after = component_input_digest(replace(config, packages=packages), "a") 115 self.assertEqual(before, after) 116 117 def test_repository_or_tag_change_changes_digest(self): 118 config = packaging_config() 119 before = component_input_digest(config, "a") 120 packages = dict(config.packages) 121 packages["a"] = replace(packages["a"], tag="v1.0.1") 122 self.assertNotEqual( 123 before, 124 component_input_digest(replace(config, packages=packages), "a"), 125 ) 126 127 128 class BuildGroupingTests(unittest.TestCase): 129 def test_workspace_packages_with_same_tag_share_group(self): 130 config = packaging_config() 131 groups = group_components(config, {"b", "c"}, ["a", "b", "c"]) 132 self.assertEqual([["b", "c"]], groups) 133 134 def test_different_tags_create_different_groups(self): 135 config = packaging_config() 136 packages = dict(config.packages) 137 packages["c"] = replace(packages["c"], tag="v2.0.1") 138 config = replace(config, packages=packages) 139 groups = group_components(config, {"b", "c"}, ["a", "b", "c"]) 140 self.assertEqual({("b",), ("c",)}, {tuple(group) for group in groups}) 141 142 def test_generic_packages_remain_isolated(self): 143 config = packaging_config(workspace=False) 144 groups = group_components(config, {"b", "c"}, ["a", "b", "c"]) 145 self.assertEqual([["b"], ["c"]], groups) 146 147 148 class ArtifactStateTests(unittest.TestCase): 149 def setUp(self): 150 self.temporary = tempfile.TemporaryDirectory() 151 root = Path(self.temporary.name) 152 self.artifacts = root / "artifacts" 153 self.state = root / "state" 154 self.staging = root / "staging" 155 self.artifacts.mkdir() 156 self.state.mkdir() 157 self.staging.mkdir() 158 159 def tearDown(self): 160 self.temporary.cleanup() 161 162 def record_old_build(self, component): 163 component_dir = self.artifacts / component 164 component_dir.mkdir() 165 (component_dir / "old.deb").write_bytes(b"old") 166 (self.state / f"{component}.key").write_text("old-key\n", encoding="utf-8") 167 168 def test_failed_group_keeps_every_previous_artifact_and_key(self): 169 config = packaging_config() 170 for component in ("b", "c"): 171 self.record_old_build(component) 172 173 def fail_build(components, staging_dir): 174 for component in components: 175 (staging_dir / component / "new.deb").write_bytes(b"new") 176 raise RuntimeError("build failed") 177 178 with self.assertRaisesRegex(RuntimeError, "build failed"): 179 rebuild_outdated( 180 ["b", "c"], 181 config, 182 {"b": "new-b", "c": "new-c"}, 183 self.artifacts, 184 self.state, 185 self.staging, 186 fail_build, 187 ) 188 189 for component in ("b", "c"): 190 self.assertEqual( 191 b"old", (self.artifacts / component / "old.deb").read_bytes() 192 ) 193 self.assertEqual( 194 "old-key", (self.state / f"{component}.key").read_text().strip() 195 ) 196 self.assertEqual([], list(self.staging.iterdir())) 197 198 def test_successful_group_promotes_each_package(self): 199 config = packaging_config() 200 201 def succeed_build(components, staging_dir): 202 for component in components: 203 (staging_dir / component / f"{component}.deb").write_bytes( 204 component.encode() 205 ) 206 207 rebuilt = rebuild_outdated( 208 ["b", "c"], 209 config, 210 {"b": "key-b", "c": "key-c"}, 211 self.artifacts, 212 self.state, 213 self.staging, 214 succeed_build, 215 ) 216 self.assertEqual(["b", "c"], rebuilt) 217 for component in ("b", "c"): 218 self.assertTrue( 219 component_is_current( 220 self.artifacts, 221 self.state, 222 component, 223 f"key-{component}", 224 ) 225 ) 226 227 228 class PackageRunnerTests(unittest.TestCase): 229 @patch("buildscripts.sandcastle_build_packages.subprocess.run") 230 def test_package_container_has_all_cache_mounts_and_group(self, run): 231 with tempfile.TemporaryDirectory() as temporary: 232 root = Path(temporary) 233 runner = podman_build_runner( 234 root=root, 235 image="builder-image", 236 architecture="amd64", 237 distro="debian-trixie", 238 codename="trixie", 239 artifact_dir=root / "artifacts", 240 cache_dir=root / "cache", 241 ) 242 runner(["b", "c"], root / "staging") 243 244 command = run.call_args.args[0] 245 self.assertIn("--ulimit=nofile=2048:2048", command) 246 self.assertEqual(["b", "c"], command[-2:]) 247 for _source, target in CACHE_MOUNTS: 248 self.assertTrue(any(f"target={target}" in argument for argument in command)) 249 self.assertTrue(any("target=/var/cache/apt/archives" in arg for arg in command)) 250 run.assert_called_once_with(command, check=True) 251 252 253 class BuildOrchestrationTests(unittest.TestCase): 254 def test_build_phases_cache_mounts_and_no_cache_propagation(self): 255 repository = Path(__file__).resolve().parents[1] 256 with tempfile.TemporaryDirectory() as temporary: 257 root = Path(temporary) 258 (root / "bin").mkdir() 259 (root / "buildscripts").mkdir() 260 shutil.copy(repository / "sandcastle-build", root / "sandcastle-build") 261 (root / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") 262 (root / "packages.toml").touch() 263 (root / "buildscripts" / "package_config.py").touch() 264 (root / "buildscripts" / "sandcastle_build_packages.py").touch() 265 log = root / "calls" 266 267 fake_command = """#!/bin/sh 268 if [ \"$1\" = \"-\" ]; then 269 printf 'git://example.test/turnstile\\tv1.0.0\\n' 270 exit 0 271 fi 272 printf '%s' \"$0\" >> \"$CALL_LOG\" 273 printf ' <%s>' \"$@\" >> \"$CALL_LOG\" 274 printf '\\n' >> \"$CALL_LOG\" 275 """ 276 for command in ("podman", "python3"): 277 executable = root / "bin" / command 278 executable.write_text(fake_command, encoding="utf-8") 279 executable.chmod(0o755) 280 git = root / "bin" / "git" 281 git.write_text("#!/bin/sh\nprintf 'sandcastle-version\\n'\n", encoding="utf-8") 282 git.chmod(0o755) 283 284 environment = dict(**__import__("os").environ) 285 environment["PATH"] = f"{root / 'bin'}:{environment['PATH']}" 286 environment["CALL_LOG"] = str(log) 287 subprocess.run( 288 [str(root / "sandcastle-build"), "--no-cache"], 289 check=True, 290 cwd=root, 291 env=environment, 292 stdout=subprocess.PIPE, 293 text=True, 294 ) 295 calls = log.read_text(encoding="utf-8").splitlines() 296 self.assertEqual(3, len(calls)) 297 self.assertIn("<--target> <base-system>", calls[0]) 298 self.assertIn("<--force>", calls[1]) 299 self.assertIn("<--target> <taler-final>", calls[2]) 300 self.assertIn("<--build-arg> <TURNSTILE_TAG=v1.0.0>", calls[2]) 301 self.assertIn("<--no-cache>", calls[0]) 302 self.assertIn("<--no-cache>", calls[2]) 303 for call in (calls[0], calls[2]): 304 self.assertIn(":/var/cache/apt/archives:z>", call) 305 self.assertIn(":/root/.npm:z>", call) 306 307 308 if __name__ == "__main__": 309 unittest.main()