commit 51b18443eb51daf07e43c809e294f5bc8af079bb parent e41ee6a7e619f0c18761afff609358e7f0c46e4e Author: Florian Dold <dold@taler.net> Date: Tue, 25 Aug 2026 22:22:38 +0200 sandcastle builds: group TypeScript packages and share caches Diffstat:
57 files changed, 1142 insertions(+), 604 deletions(-)
diff --git a/Dockerfile b/Dockerfile @@ -76,11 +76,12 @@ RUN DEBIAN_FRONTEND=noninteractive \ RUN npm install -g pnpm@10 FROM base-system as turnstile -COPY buildconfig/turnstile.* /buildconfig/ -RUN TAG=$(cat /buildconfig/turnstile.tag) && \ +ARG TURNSTILE_GIT_URL=git://git.taler.net/turnstile +ARG TURNSTILE_TAG=v0.0.2 +RUN \ cd /opt/ && \ - git clone git://git.taler.net/turnstile \ - --branch $TAG + git clone "$TURNSTILE_GIT_URL" \ + --branch "$TURNSTILE_TAG" # Final image FROM base-system as taler-final diff --git a/README.md b/README.md @@ -34,9 +34,13 @@ port for each of the services. # Upgrading Component Versions -Run `./sandcastle-upgrade` to update the git tags in `buildconfig/$component.tag` -to the latest upstream tag of the respective repository -(taken from `buildconfig/$component.giturl`). +`packages.toml` is the single source of truth for package repositories, tags, +Debian paths, dependencies, enabled state, and builders. It uses the same +`[repositories]` and `[packages]` layout as +`taler-deployment/packaging/ng/packages.toml`. + +Run `./sandcastle-upgrade` to update package tags in `packages.toml` to the +latest upstream tag of the respective repository. By default only production tags (`vX.Y.Z`, `deb-vX.Y.Z-R`) are considered. Pass `--dev` to also consider dev tags (`vX.Y.Z-dev.N`). @@ -45,30 +49,37 @@ Individual components can be given as arguments, e.g. ./sandcastle-upgrade --dev taler-exchange gnunet -Components whose tag file pins something that is not a version tag -(such as a branch name) are left alone, as are components without a -`.giturl` file. - -To only *show* the latest upstream versions without changing anything, -use `./print-latest-versions`. +Packages whose configured tag is not a supported version tag are left alone. +Use `./sandcastle-upgrade --dry` to show available updates without changing the +configuration. # Building the Container Image -1. In `buildconfig/$component.tag` set the git tag you want to build. +1. Set the package's `tag` in `packages.toml` to the git tag you want to build. 2. Run `./sandcastle-build` to build the Taler container. The resulting container will be tagged as `taler-base-all`. Package builds are incremental. Built Debian packages and their build keys are kept below `packages/`, while downloaded build data is kept below - `cache/`. A component is rebuilt when its build configuration, package - builder, generic build logic, or one of its package dependencies changes. + `cache/`. A component is rebuilt when its resolved package configuration, + build logic, builder image, or one of its package dependencies changes. Unrelated packages remain untouched. Failed builds are not recorded as current, so they are retried on the next invocation. + Packages using the `pnpm-workspace` builder are grouped by repository and + tag. Each group clones and bootstraps the monorepo once, performs one + filtered pnpm install and build for all selected workspace dependency + closures, and then creates the individual Debian packages from the prebuilt + artifacts. + + Apt, pnpm, Gradle, Cargo, npm, Go, and pip caches are shared between package + containers. Source checkouts and package build trees remain fresh. + Use `./sandcastle-build --no-cache` to rebuild the package builder, every - component package, and the final image without cached image layers. + component package, and the final image without cached image layers. Shared + dependency and compiler caches are retained. By default `sandcastle-build` builds from the `Dockerfile` in the same directory as the script. diff --git a/buildconfig/README b/buildconfig/README @@ -1,5 +0,0 @@ -These files determine the git tag from which the respective components are -built in the base Docker image. - -They are in separate files to make modification checking with -staged Docker builds work nicely. diff --git a/buildconfig/challenger-webui.debpath b/buildconfig/challenger-webui.debpath @@ -1 +0,0 @@ -packages/challenger-webui diff --git a/buildconfig/challenger-webui.giturl b/buildconfig/challenger-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/buildconfig/challenger-webui.tag b/buildconfig/challenger-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.6 diff --git a/buildconfig/challenger.giturl b/buildconfig/challenger.giturl @@ -1 +0,0 @@ -git://git.taler.net/challenger diff --git a/buildconfig/challenger.tag b/buildconfig/challenger.tag @@ -1 +0,0 @@ -v1.6.3-dev.1 diff --git a/buildconfig/donau.giturl b/buildconfig/donau.giturl @@ -1 +0,0 @@ -git://git.taler.net/donau.git diff --git a/buildconfig/donau.tag b/buildconfig/donau.tag @@ -1 +0,0 @@ -v1.6.7 diff --git a/buildconfig/gnunet.giturl b/buildconfig/gnunet.giturl @@ -1 +0,0 @@ -git://git.gnunet.org/gnunet.git diff --git a/buildconfig/gnunet.tag b/buildconfig/gnunet.tag @@ -1 +0,0 @@ -v0.28.1-dev.5 diff --git a/buildconfig/libeufin-bank-webui.debpath b/buildconfig/libeufin-bank-webui.debpath @@ -1 +0,0 @@ -packages/libeufin-bank-webui diff --git a/buildconfig/libeufin-bank-webui.giturl b/buildconfig/libeufin-bank-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/buildconfig/libeufin-bank-webui.tag b/buildconfig/libeufin-bank-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.6 diff --git a/buildconfig/libeufin.giturl b/buildconfig/libeufin.giturl @@ -1 +0,0 @@ -git://git.taler.net/libeufin diff --git a/buildconfig/libeufin.tag b/buildconfig/libeufin.tag @@ -1 +0,0 @@ -v1.6.9-dev.1 diff --git a/buildconfig/paivana-httpd.giturl b/buildconfig/paivana-httpd.giturl @@ -1 +0,0 @@ -git://git.taler.net/paivana diff --git a/buildconfig/paivana-httpd.tag b/buildconfig/paivana-httpd.tag @@ -1 +0,0 @@ -v1.6.4 diff --git a/buildconfig/sync.tag b/buildconfig/sync.tag @@ -1 +0,0 @@ -v1.6.0-dev.1 diff --git a/buildconfig/taler-auditor-webui.debpath b/buildconfig/taler-auditor-webui.debpath @@ -1 +0,0 @@ -packages/taler-auditor-webui diff --git a/buildconfig/taler-auditor-webui.giturl b/buildconfig/taler-auditor-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/buildconfig/taler-auditor-webui.tag b/buildconfig/taler-auditor-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.6 diff --git a/buildconfig/taler-directory.giturl b/buildconfig/taler-directory.giturl @@ -1 +0,0 @@ -git://git.gnunet.org/taldir diff --git a/buildconfig/taler-directory.tag b/buildconfig/taler-directory.tag @@ -1 +0,0 @@ -v1.5.1 diff --git a/buildconfig/taler-exchange.giturl b/buildconfig/taler-exchange.giturl @@ -1 +0,0 @@ -git://git.taler.net/exchange.git diff --git a/buildconfig/taler-exchange.tag b/buildconfig/taler-exchange.tag @@ -1 +0,0 @@ -v1.6.15-dev.1 diff --git a/buildconfig/taler-harness.debpath b/buildconfig/taler-harness.debpath @@ -1 +0,0 @@ -packages/taler-harness diff --git a/buildconfig/taler-harness.giturl b/buildconfig/taler-harness.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/buildconfig/taler-harness.tag b/buildconfig/taler-harness.tag @@ -1 +0,0 @@ -v1.6.33-dev.6 diff --git a/buildconfig/taler-mailbox.giturl b/buildconfig/taler-mailbox.giturl @@ -1 +0,0 @@ -git://git.gnunet.org/taler-mailbox diff --git a/buildconfig/taler-mailbox.tag b/buildconfig/taler-mailbox.tag @@ -1 +0,0 @@ -v1.5.1 diff --git a/buildconfig/taler-merchant-demos.giturl b/buildconfig/taler-merchant-demos.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-merchant-demos diff --git a/buildconfig/taler-merchant-demos.tag b/buildconfig/taler-merchant-demos.tag @@ -1 +0,0 @@ -v1.6.3 diff --git a/buildconfig/taler-merchant-webui.debpath b/buildconfig/taler-merchant-webui.debpath @@ -1 +0,0 @@ -packages/taler-merchant-webui diff --git a/buildconfig/taler-merchant-webui.giturl b/buildconfig/taler-merchant-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/buildconfig/taler-merchant-webui.tag b/buildconfig/taler-merchant-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.6 diff --git a/buildconfig/taler-merchant.giturl b/buildconfig/taler-merchant.giturl @@ -1 +0,0 @@ -git://git.taler.net/merchant.git diff --git a/buildconfig/taler-merchant.tag b/buildconfig/taler-merchant.tag @@ -1 +0,0 @@ -v1.6.19 diff --git a/buildconfig/taler-wallet-cli.debpath b/buildconfig/taler-wallet-cli.debpath @@ -1 +0,0 @@ -packages/taler-wallet-cli diff --git a/buildconfig/taler-wallet-cli.giturl b/buildconfig/taler-wallet-cli.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/buildconfig/taler-wallet-cli.tag b/buildconfig/taler-wallet-cli.tag @@ -1 +0,0 @@ -v1.6.33-dev.6 diff --git a/buildconfig/turnstile.tag b/buildconfig/turnstile.tag @@ -1 +0,0 @@ -v0.0.2 diff --git a/buildscripts/package_config.py b/buildscripts/package_config.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# This file is in the public domain. + +"""Load, validate, and atomically update Sandcastle package configuration.""" + +from __future__ import annotations + +import json +import os +import stat +import tempfile +import tomllib +from dataclasses import dataclass +from pathlib import Path + + +class ConfigError(ValueError): + pass + + +SUPPORTED_BUILDERS = {"generic", "pnpm-workspace"} + + +@dataclass(frozen=True) +class RepositoryConfig: + name: str + url: str + builder: str = "generic" + + +@dataclass(frozen=True) +class PackageConfig: + name: str + repository: str + tag: str + debian_path: str = "" + dependencies: tuple[str, ...] = () + enabled: bool = True + builder: str | None = None + + +@dataclass(frozen=True) +class PackagingConfig: + repositories: dict[str, RepositoryConfig] + packages: dict[str, PackageConfig] + + def repository_for(self, package: PackageConfig) -> RepositoryConfig: + return self.repositories[package.repository] + + def builder_for(self, package: PackageConfig) -> str: + return package.builder or self.repository_for(package).builder + + def enabled_packages(self) -> list[str]: + return sorted(name for name, package in self.packages.items() if package.enabled) + + +def _expect_table(value: object, description: str) -> dict: + if not isinstance(value, dict): + raise ConfigError(f"{description} must be a table") + return value + + +def _reject_unknown(table: dict, allowed: set[str], description: str) -> None: + unknown = sorted(set(table) - allowed) + if unknown: + raise ConfigError(f"unknown field(s) in {description}: {', '.join(unknown)}") + + +def _required_string(table: dict, field: str, description: str) -> str: + value = table.get(field) + if not isinstance(value, str) or not value: + raise ConfigError(f"{description}.{field} must be a non-empty string") + return value + + +def _optional_string( + table: dict, field: str, default: str, description: str +) -> str: + value = table.get(field, default) + if not isinstance(value, str): + raise ConfigError(f"{description}.{field} must be a string") + return value + + +def parse_config(data: object) -> PackagingConfig: + root = _expect_table(data, "configuration") + _reject_unknown(root, {"repositories", "packages"}, "configuration") + repository_tables = _expect_table(root.get("repositories"), "repositories") + package_tables = _expect_table(root.get("packages"), "packages") + + repositories: dict[str, RepositoryConfig] = {} + for name, raw_repository in repository_tables.items(): + description = f"repositories.{name}" + repository = _expect_table(raw_repository, description) + _reject_unknown(repository, {"url", "builder"}, description) + builder = _optional_string(repository, "builder", "generic", description) + if builder not in SUPPORTED_BUILDERS: + raise ConfigError(f"{description}.builder is not supported: {builder!r}") + repositories[name] = RepositoryConfig( + name=name, + url=_required_string(repository, "url", description), + builder=builder, + ) + + packages: dict[str, PackageConfig] = {} + for name, raw_package in package_tables.items(): + description = f"packages.{name}" + package = _expect_table(raw_package, description) + _reject_unknown( + package, + { + "repository", + "tag", + "debian_path", + "dependencies", + "enabled", + "builder", + }, + description, + ) + repository = _required_string(package, "repository", description) + if repository not in repositories: + raise ConfigError(f"{description} references unknown repository {repository!r}") + dependencies = package.get("dependencies", []) + if not isinstance(dependencies, list) or not all( + isinstance(dependency, str) and dependency for dependency in dependencies + ): + raise ConfigError(f"{description}.dependencies must be an array of strings") + if len(set(dependencies)) != len(dependencies): + raise ConfigError(f"{description}.dependencies contains duplicates") + enabled = package.get("enabled", True) + if not isinstance(enabled, bool): + raise ConfigError(f"{description}.enabled must be a boolean") + builder = package.get("builder") + if builder is not None and (not isinstance(builder, str) or not builder): + raise ConfigError(f"{description}.builder must be a non-empty string") + if builder is not None and builder not in SUPPORTED_BUILDERS: + raise ConfigError(f"{description}.builder is not supported: {builder!r}") + packages[name] = PackageConfig( + name=name, + repository=repository, + tag=_required_string(package, "tag", description), + debian_path=_optional_string(package, "debian_path", "", description), + dependencies=tuple(dependencies), + enabled=enabled, + builder=builder, + ) + + for package in packages.values(): + for dependency in package.dependencies: + if dependency not in packages: + raise ConfigError( + f"packages.{package.name} references unknown dependency {dependency!r}" + ) + _check_dependency_cycles(packages) + return PackagingConfig(repositories=repositories, packages=packages) + + +def _check_dependency_cycles(packages: dict[str, PackageConfig]) -> None: + permanent: set[str] = set() + temporary: list[str] = [] + + def visit(name: str) -> None: + if name in permanent: + return + if name in temporary: + cycle = temporary[temporary.index(name) :] + [name] + raise ConfigError(f"package dependency cycle: {' -> '.join(cycle)}") + temporary.append(name) + for dependency in packages[name].dependencies: + visit(dependency) + temporary.pop() + permanent.add(name) + + for name in sorted(packages): + visit(name) + + +def load_config(path: str | Path) -> PackagingConfig: + path = Path(path) + try: + with path.open("rb") as config_file: + return parse_config(tomllib.load(config_file)) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise ConfigError(f"unable to read {path}: {exc}") from exc + + +def _toml_string(value: str) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _toml_string_array(values: tuple[str, ...]) -> str: + return "[" + ", ".join(_toml_string(value) for value in values) + "]" + + +def serialize_config(config: PackagingConfig) -> str: + lines = [ + "# Package definitions for Sandcastle.", + "# This file is rewritten in canonical form by `sandcastle-upgrade`.", + "", + ] + for name in sorted(config.repositories): + repository = config.repositories[name] + lines.extend( + [ + f"[repositories.{_toml_string(name)}]", + f"url = {_toml_string(repository.url)}", + f"builder = {_toml_string(repository.builder)}", + "", + ] + ) + for name in sorted(config.packages): + package = config.packages[name] + lines.extend( + [ + f"[packages.{_toml_string(name)}]", + f"repository = {_toml_string(package.repository)}", + f"tag = {_toml_string(package.tag)}", + f"debian_path = {_toml_string(package.debian_path)}", + f"dependencies = {_toml_string_array(package.dependencies)}", + f"enabled = {'true' if package.enabled else 'false'}", + ] + ) + if package.builder is not None: + lines.append(f"builder = {_toml_string(package.builder)}") + lines.append("") + return "\n".join(lines) + + +def write_config(path: str | Path, config: PackagingConfig) -> None: + path = Path(path) + serialized = serialize_config(config) + reparsed = parse_config(tomllib.loads(serialized)) + if reparsed != config: + raise ConfigError("serialized configuration did not round-trip") + + mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else 0o644 + fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as config_file: + config_file.write(serialized) + config_file.flush() + os.fsync(config_file.fileno()) + os.replace(temporary_name, path) + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise diff --git a/buildscripts/sandcastle-build-generic b/buildscripts/sandcastle-build-generic @@ -1,175 +1,186 @@ #!/usr/bin/env python3 # This file is in the public domain. -# Helper script to build the latest DEB packages in the container. +"""Build one generic package or a group from one pnpm workspace.""" -# Supported tag syntax variants: -# v$maj.$min$.$patch => release version -# v$maj.$min$.$patch-dev.$n => dev version -# deb-v$maj.$min$.$patch-$revision => release version with debian revision -# Debian revisions of dev versions are *not* supported +from __future__ import annotations -import glob +import json import os import shutil import subprocess import sys from email.utils import formatdate -from logging import warning +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from package_config import ConfigError, load_config def run_cmd(cmd, shell=False, cwd=None, env=None): - """Helper to run commands and exit on failure (mimicking set -e).""" - # If specific env vars are passed, update the current env, otherwise use current command_env = os.environ.copy() if env: command_env.update(env) - - # Flush stdout so logs appear in order sys.stdout.flush() - subprocess.check_call(cmd, shell=shell, cwd=cwd, env=command_env) def get_tag_debver(tag): - """Get a debian version string from a git tag""" if tag.startswith("v"): devsuff = "-dev." - d = tag.find(devsuff) - if d < 0: + position = tag.find(devsuff) + if position < 0: return tag[1:] - else: - return tag[1:d] + "~dev" + tag[d + len(devsuff) :] + return tag[1:position] + "~dev" + tag[position + len(devsuff) :] if tag.startswith("deb-v"): tag = tag[5:] if "-" in tag: - upstream, revision = tag.split("-", 1) - return upstream + "-" + revision + version, revision = tag.split("-", 1) + return version + "-" + revision return tag - warning(f"unexpected tag format: {tag}") - return "100.0.0" # Version big enough to work every time + raise ValueError(f"unexpected tag format: {tag}") def make_codename_version(deb_version, codename): - """Make locally-built packages distinct from other distro builds.""" if "-" in deb_version: return f"{deb_version}+{codename}" return f"{deb_version}-0+{codename}" -def main(): - if "LD_LIBRARY_PATH" in os.environ: - del os.environ["LD_LIBRARY_PATH"] - - # Arguments - if len(sys.argv) != 4: - print( - f"Usage: {sys.argv[0]} <PACKAGE> <CODENAME> <ARCH>", - file=sys.stderr, - ) - sys.exit(1) - - package = sys.argv[1] - codename = sys.argv[2] - architecture = sys.argv[3] - - # Path of the debian/ folder in the repository - DEBIANPATH = "" - debpath_file = f"/buildconfig/{package}.debpath" - if os.path.exists(debpath_file): - with open(debpath_file, "r") as f: - DEBIANPATH = f.read().strip() - - print( - f"Building {package} for {codename}/{architecture} with generic build logic", - file=sys.stderr, - ) - - with open(f"/buildconfig/{package}.tag", "r") as f: - tag = f.read().strip() - - outdir = "/out" - os.makedirs(outdir, exist_ok=True) - +def scan_local_repository(): os.chdir("/pkgdir") - run_cmd( - [ - "bash", - "-o", - "pipefail", - "-c", - "dpkg-scanpackages . /dev/null | xz - > Packages.xz", - ] + ["bash", "-o", "pipefail", "-c", "dpkg-scanpackages . /dev/null | xz - > Packages.xz"] + ) + Path("/etc/apt/sources.list.d/taler-packaging-local.list").write_text( + "deb [trusted=yes] file:/pkgdir ./\n", encoding="utf-8" ) - - with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as f: - f.write("deb [trusted=yes] file:/pkgdir ./\n") - run_cmd(["apt-get", "update"]) - # Prepare Build Directory - if not os.path.exists("/build"): - os.makedirs("/build") - os.chdir("/build") - - with open(f"/buildconfig/{package}.giturl", "r") as f: - GITURL = f.read().strip() - - run_cmd(["git", "config", "--global", "advice.detachedHead", "false"]) - run_cmd(["git", "clone", "--depth=1", f"--branch={tag}", GITURL, package]) - - build_pkg_path = os.path.join("/build", package, DEBIANPATH) - os.chdir(build_pkg_path) - - deb_version = make_codename_version(get_tag_debver(tag), codename) - # Bootstrap and Install Deps - os.chdir(os.path.join("/build", package)) - run_cmd(["./bootstrap"]) - - os.chdir(build_pkg_path) - - # Install build-time dependencies - tool_cmd = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" - run_cmd(["mk-build-deps", "--install", f"--tool={tool_cmd}", "debian/control"]) - - # Sparse checkout hint - with open(".version", "w") as f: - f.write(f"{deb_version}\n") - - # Configure Environment for Build - deb_dbg_repo = "debian/.debhelper/" - os.environ["DEB_DBG_SYMBOLS_REPO"] = deb_dbg_repo - os.makedirs(deb_dbg_repo, exist_ok=True) +def validate_group(config, names): + if not names or len(set(names)) != len(names): + raise ConfigError("packages must be specified once") + packages = [] + keys = set() + for name in names: + try: + package = config.packages[name] + except KeyError as exc: + raise ConfigError(f"unknown package {name!r}") from exc + repository = config.repository_for(package) + builder = config.builder_for(package) + keys.add((repository.url, package.tag, builder)) + packages.append(package) + if len(keys) != 1: + raise ConfigError("grouped packages must have the same repository, tag, and builder") + repository_url, tag, builder = keys.pop() + if builder == "generic" and len(packages) != 1: + raise ConfigError("the generic builder accepts exactly one package") + return packages, repository_url, tag, builder + + +def install_build_dependencies(source_dir, package_paths): + tool = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" + controls = [str(package_path / "debian" / "control") for package_path in package_paths] + run_cmd(["mk-build-deps", "--install", f"--tool={tool}", *controls], cwd=source_dir) + + +def prepare_pnpm_workspace(source_dir, package_paths): + filters = [] + for package_path in package_paths: + package_json = package_path / "package.json" + with package_json.open(encoding="utf-8") as package_file: + workspace_name = json.load(package_file).get("name") + if not isinstance(workspace_name, str) or not workspace_name: + raise ConfigError(f"{package_json} has no package name") + filters.extend(["--filter", f"{workspace_name}..."]) + run_cmd(["pnpm", "install", "--frozen-lockfile", *filters], cwd=source_dir) + run_cmd(["pnpm", "run", *filters, "build"], cwd=source_dir) + + +def write_changelog(package_path, package_name, version): + changelog = f"""\ +{package_name} ({version}) unstable; urgency=low - open(os.path.join(deb_dbg_repo, "debian-symbols-pool"), "a").close() + * Release {version} (for sandcastle-ng). - os.environ["DEB_BUILD_MAINT_OPTIONS"] = "debug" + -- Taler Packaging Team <deb@taler.net> {formatdate(localtime=True)} +""" + (package_path / "debian" / "changelog").write_text(changelog, encoding="utf-8") + + +def build_package(package, package_path, version, prebuilt): + write_changelog(package_path, package.name, version) + debug_repository = package_path / "debian" / ".debhelper" + debug_repository.mkdir(parents=True, exist_ok=True) + (debug_repository / "debian-symbols-pool").touch() + environment = { + "DEB_BUILD_MAINT_OPTIONS": "debug", + "DEB_DBG_SYMBOLS_REPO": "debian/.debhelper/", + } + if prebuilt: + environment["TALER_PACKAGING_PREBUILT"] = "1" + + output_dir = package_path.parent + before = set(output_dir.glob("*.deb")) + run_cmd( + ["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"], + cwd=package_path, + env=environment, + ) + deb_files = sorted(set(output_dir.glob("*.deb")) - before) + if not deb_files: + raise RuntimeError(f"build of {package.name} produced no .deb packages") + destination = Path("/out") / package.name + for deb_file in deb_files: + shutil.copy(deb_file, destination) - debian_date = formatdate(localtime=True) - changelog = f"""\ -{package} ({deb_version}) unstable; urgency=low +def build_packages(codename, architecture, names): + del architecture # Package metadata determines whether outputs are arch-specific. + os.environ.pop("LD_LIBRARY_PATH", None) + config = load_config("/packages.toml") + packages, repository_url, tag, builder = validate_group(config, names) + print(f"Building {' '.join(names)} with {builder} build logic", file=sys.stderr) - * Release {deb_version} (for sandcastle-ng). + scan_local_repository() + source_dir = Path("/build/source") + source_dir.parent.mkdir(parents=True, exist_ok=True) + run_cmd(["git", "config", "--global", "advice.detachedHead", "false"]) + run_cmd( + ["git", "clone", "--depth=1", f"--branch={tag}", repository_url, str(source_dir)] + ) + run_cmd(["./bootstrap"], cwd=source_dir) - -- Taler Packaging Team <deb@taler.net> {debian_date} -""" + version = make_codename_version(get_tag_debver(tag), codename) + package_paths = [source_dir / package.debian_path for package in packages] + for package, package_path in zip(packages, package_paths): + control = package_path / "debian" / "control" + if not control.is_file(): + raise ConfigError(f"{package.name} has no debian/control at tag {tag}: {control}") + (package_path / ".version").write_text(version + "\n", encoding="utf-8") - with open("debian/changelog", "w") as f: - f.write(changelog) + install_build_dependencies(source_dir, package_paths) + if builder == "pnpm-workspace": + prepare_pnpm_workspace(source_dir, package_paths) + for package, package_path in zip(packages, package_paths): + build_package(package, package_path, version, builder == "pnpm-workspace") - # Build Package - run_cmd(["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"]) - # Copy binary packages into the host-side staging directory. Debug packages - # are intentionally omitted from the final sandcastle image. - deb_files = glob.glob("../*.deb") - if not deb_files: - raise RuntimeError(f"build of {package} produced no .deb packages") - for deb_file in deb_files: - shutil.copy(deb_file, outdir) +def main(): + if len(sys.argv) < 4: + print( + f"Usage: {sys.argv[0]} <CODENAME> <ARCH> <PACKAGE>...", + file=sys.stderr, + ) + return 1 + try: + build_packages(sys.argv[1], sys.argv[2], sys.argv[3:]) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 1 + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/buildscripts/sandcastle_build_packages.py b/buildscripts/sandcastle_build_packages.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # This file is in the public domain. -"""Build sandcastle Debian packages incrementally in fresh containers.""" +"""Build outdated Sandcastle Debian packages in fresh containers.""" from __future__ import annotations @@ -15,39 +15,15 @@ import uuid from collections.abc import Callable, Iterable, Mapping, Sequence from pathlib import Path +try: + from .package_config import PackagingConfig, load_config +except ImportError: # Direct script execution inside the builder container. + from package_config import PackagingConfig, load_config + DISTRO = "debian-trixie" BUILDER_IMAGE = "localhost/taler-sandcastle-package-builder:latest" - -# Keep this list in sync with the packages installed by the final image. -COMPONENTS = ( - "gnunet", - "taler-directory", - "taler-mailbox", - "taler-exchange", - "taler-auditor-webui", - "donau", - "taler-merchant", - "paivana-httpd", - "challenger-webui", - "challenger", - "libeufin", - "taler-merchant-demos", - "taler-wallet-cli", - "taler-harness", - "taler-merchant-webui", - "libeufin-bank-webui", -) - -DEPENDENCIES = { - "taler-exchange": ("gnunet",), - "donau": ("gnunet", "taler-exchange"), - "taler-merchant": ("gnunet", "taler-exchange", "donau"), - "challenger": ("taler-exchange",), - "paivana-httpd": ("gnunet", "taler-exchange", "taler-merchant"), -} - -BuildRunner = Callable[[str, Path], None] +BuildRunner = Callable[[Sequence[str], Path], None] def topological_order( @@ -57,7 +33,6 @@ def topological_order( component_set = set(components) if len(component_set) != len(components): raise ValueError("component list contains duplicates") - order: list[str] = [] permanent: set[str] = set() temporary: set[str] = set() @@ -68,8 +43,7 @@ def topological_order( if component in temporary: raise ValueError(f"dependency cycle involving {component}") if component not in component_set: - raise ValueError(f"unknown component in dependency graph: {component}") - + raise ValueError(f"unknown or disabled package dependency: {component}") temporary.add(component) for dependency in dependencies.get(component, ()): visit(dependency) @@ -82,6 +56,10 @@ def topological_order( return order +def canonical_json(value: object) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + def file_digest(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as source: @@ -90,21 +68,28 @@ def file_digest(path: Path) -> str: return digest.hexdigest() -def component_input_digest(config_dir: Path, component: str) -> str: - """Hash all build configuration that belongs to one component.""" - inputs: dict[str, str | None] = {} - required_suffixes = ("tag", "giturl") - optional_suffixes = ("debpath", "checkout") - for suffix in required_suffixes + optional_suffixes: - path = config_dir / f"{component}.{suffix}" - if suffix in required_suffixes and not path.is_file(): - raise FileNotFoundError(f"missing build configuration: {path}") - inputs[suffix] = file_digest(path) if path.is_file() else None +def files_digest(paths: Iterable[Path]) -> str: + inputs = { + path.name: file_digest(path) + for path in sorted(paths) + if path.is_file() and not path.name.startswith("__pycache__") + } return hashlib.sha256(canonical_json(inputs)).hexdigest() -def canonical_json(value: object) -> bytes: - return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() +def component_input_digest(config: PackagingConfig, component: str) -> str: + """Hash the resolved configuration that belongs to one package.""" + package = config.packages[component] + repository = config.repository_for(package) + inputs = { + "repository_url": repository.url, + "builder": config.builder_for(package), + "tag": package.tag, + "debian_path": package.debian_path, + "dependencies": package.dependencies, + "enabled": package.enabled, + } + return hashlib.sha256(canonical_json(inputs)).hexdigest() def desired_build_keys( @@ -115,19 +100,19 @@ def desired_build_keys( distro: str, architecture: str, builder_image_id: str, - build_script_digest: str, + build_scripts_digest: str, ) -> dict[str, str]: """Compute recursive keys so dependency changes invalidate consumers.""" keys: dict[str, str] = {} for component in order: payload = { - "schema": 1, + "schema": 2, "component": component, "component_input": component_inputs[component], "distro": distro, "architecture": architecture, "builder_image_id": builder_image_id, - "build_script": build_script_digest, + "build_scripts": build_scripts_digest, "dependencies": { dependency: keys[dependency] for dependency in sorted(dependencies.get(component, ())) @@ -137,6 +122,57 @@ def desired_build_keys( return keys +def component_group_key(config: PackagingConfig, component: str) -> tuple[str, ...]: + package = config.packages[component] + repository = config.repository_for(package) + builder = config.builder_for(package) + if builder == "generic": + return repository.url, package.tag, builder, component + return repository.url, package.tag, builder, "" + + +def group_components( + config: PackagingConfig, selected: Iterable[str], build_order: Sequence[str] +) -> list[list[str]]: + """Group selected packages without violating package dependency order.""" + selected_set = set(selected) + groups: dict[tuple[str, ...], list[str]] = {} + component_to_group: dict[str, tuple[str, ...]] = {} + for component in build_order: + if component not in selected_set: + continue + key = component_group_key(config, component) + groups.setdefault(key, []).append(component) + component_to_group[component] = key + + group_dependencies = {key: set() for key in groups} + for component, key in component_to_group.items(): + for dependency in config.packages[component].dependencies: + dependency_key = component_to_group.get(dependency) + if dependency_key is not None and dependency_key != key: + group_dependencies[key].add(dependency_key) + + ordered: list[list[str]] = [] + permanent: set[tuple[str, ...]] = set() + temporary: set[tuple[str, ...]] = set() + + def visit(key: tuple[str, ...]) -> None: + if key in permanent: + return + if key in temporary: + raise ValueError("build groups contain a dependency cycle") + temporary.add(key) + for dependency in sorted(group_dependencies[key]): + visit(dependency) + temporary.remove(key) + permanent.add(key) + ordered.append(groups[key]) + + for key in groups: + visit(key) + return ordered + + def state_path(state_dir: Path, component: str) -> Path: return state_dir / f"{component}.key" @@ -148,9 +184,8 @@ def package_files(artifact_dir: Path, component: str) -> list[Path]: def component_is_current( artifact_dir: Path, state_dir: Path, component: str, desired_key: str ) -> bool: - marker = state_path(state_dir, component) try: - recorded_key = marker.read_text(encoding="utf-8").strip() + recorded_key = state_path(state_dir, component).read_text(encoding="utf-8").strip() except FileNotFoundError: return False return recorded_key == desired_key and bool(package_files(artifact_dir, component)) @@ -164,11 +199,10 @@ def write_state_atomically(path: Path, desired_key: str) -> None: def promote_artifacts(staging_dir: Path, artifact_dir: Path, component: str) -> None: - """Replace a component's artifacts while retaining the old set on failure.""" + """Replace one package's artifacts while retaining the old set on failure.""" destination = artifact_dir / component artifact_dir.mkdir(parents=True, exist_ok=True) backup = staging_dir.with_name(f".{component}.{uuid.uuid4().hex}.old") - if destination.exists(): os.replace(destination, backup) try: @@ -183,6 +217,7 @@ def promote_artifacts(staging_dir: Path, artifact_dir: Path, component: str) -> def rebuild_outdated( order: Sequence[str], + config: PackagingConfig, desired_keys: Mapping[str, str], artifact_dir: Path, state_dir: Path, @@ -191,27 +226,45 @@ def rebuild_outdated( *, force: bool = False, ) -> list[str]: - """Build outdated components in order and promote only successful outputs.""" - rebuilt: list[str] = [] + """Build outdated packages in groups and promote only complete group outputs.""" + selected = [ + component + for component in order + if force + or not component_is_current( + artifact_dir, state_dir, component, desired_keys[component] + ) + ] + selected_set = set(selected) for component in order: - desired_key = desired_keys[component] - if not force and component_is_current( - artifact_dir, state_dir, component, desired_key - ): - print(f"Package {component}: up to date") - continue + status = "rebuilding" if component in selected_set else "up to date" + print(f"Package {component}: {status}") - print(f"Package {component}: rebuilding") - staging_dir = staging_root / f"{component}.{uuid.uuid4().hex}" + rebuilt: list[str] = [] + for group in group_components(config, selected, order): + group_name = "+".join(group) + staging_dir = staging_root / f"{group_name}.{uuid.uuid4().hex}" staging_dir.mkdir(parents=True, exist_ok=False) + for component in group: + (staging_dir / component).mkdir() try: - run_build(component, staging_dir) - outputs = sorted(staging_dir.glob("*.deb")) - if not outputs: - raise RuntimeError(f"build of {component} produced no .deb packages") - promote_artifacts(staging_dir, artifact_dir, component) - write_state_atomically(state_path(state_dir, component), desired_key) - rebuilt.append(component) + run_build(group, staging_dir) + missing = [ + component + for component in group + if not sorted((staging_dir / component).glob("*.deb")) + ] + if missing: + raise RuntimeError( + "build produced no .deb packages for: " + ", ".join(missing) + ) + for component in group: + promote_artifacts(staging_dir / component, artifact_dir, component) + for component in group: + write_state_atomically( + state_path(state_dir, component), desired_keys[component] + ) + rebuilt.extend(group) finally: if staging_dir.exists(): shutil.rmtree(staging_dir) @@ -236,6 +289,19 @@ def ensure_directories(paths: Iterable[Path]) -> None: path.mkdir(parents=True, exist_ok=True) +CACHE_MOUNTS = ( + ("gradle", "/root/.gradle/caches"), + ("pnpm", "/root/.local/share/pnpm/store"), + ("cargo-registry", "/root/.cargo/registry"), + ("cargo-git", "/root/.cargo/git"), + ("cargo-build", "/root/.cargo-build"), + ("npm", "/root/.npm"), + ("go-build", "/root/.cache/go-build"), + ("go-mod", "/root/go/pkg/mod"), + ("pip", "/root/.cache/pip"), +) + + def podman_build_runner( *, root: Path, @@ -247,18 +313,13 @@ def podman_build_runner( cache_dir: Path, ) -> BuildRunner: buildscripts_dir = root / "buildscripts" - buildconfig_dir = root / "buildconfig" - mounts = ( - (cache_dir / "gradle", "/root/.gradle/caches"), - (cache_dir / "pnpm", "/root/.local/share/pnpm/store"), - (cache_dir / "cargo-registry", "/root/.cargo/registry"), - (cache_dir / "cargo-git", "/root/.cargo/git"), - (cache_dir / "cargo-build", "/root/.cargo-build"), + config_path = root / "packages.toml" + mounts = tuple((cache_dir / source, target) for source, target in CACHE_MOUNTS) + ( (cache_dir / distro / "apt-archives", "/var/cache/apt/archives"), (cache_dir / distro / "apt-lists", "/var/lib/apt/lists"), ) - def run_build(component: str, staging_dir: Path) -> None: + def run_build(components: Sequence[str], staging_dir: Path) -> None: command = [ "podman", "run", @@ -279,16 +340,16 @@ def podman_build_runner( "--mount", f"type=bind,source={buildscripts_dir},target=/buildscripts,readonly", "--mount", - f"type=bind,source={buildconfig_dir},target=/buildconfig,readonly", + f"type=bind,source={config_path},target=/packages.toml,readonly", "--mount", f"type=bind,source={artifact_dir},target=/pkgdir", "--mount", f"type=bind,source={staging_dir},target=/out", image, "/buildscripts/sandcastle-build-generic", - component, codename, architecture, + *components, ] ) subprocess.run(command, check=True) @@ -307,8 +368,7 @@ def parse_args(argv: Sequence[str]) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: args = parse_args(sys.argv[1:] if argv is None else argv) root = Path(__file__).resolve().parents[1] - config_dir = root / "buildconfig" - generic_script = root / "buildscripts" / "sandcastle-build-generic" + config = load_config(root / "packages.toml") artifact_dir = root / "packages" / args.distro state_dir = root / "packages" / ".state" / args.distro staging_root = root / "packages" / ".staging" / args.distro @@ -317,38 +377,41 @@ def main(argv: Sequence[str] | None = None) -> int: vendor, separator, codename = args.distro.partition("-") if not separator or vendor != "debian" or not codename: raise ValueError(f"unsupported distribution: {args.distro}") - builder_image_id, architecture = inspect_builder(args.builder_image) artifact_dir /= architecture state_dir /= architecture staging_root /= architecture - cache_paths = ( - cache_dir / "gradle", - cache_dir / "pnpm", - cache_dir / "cargo-registry", - cache_dir / "cargo-git", - cache_dir / "cargo-build", - cache_dir / args.distro / "apt-archives" / "partial", - cache_dir / args.distro / "apt-lists" / "partial", - artifact_dir, - state_dir, - staging_root, + cache_paths = [cache_dir / source for source, _target in CACHE_MOUNTS] + cache_paths.extend( + [ + cache_dir / args.distro / "apt-archives" / "partial", + cache_dir / args.distro / "apt-lists" / "partial", + artifact_dir, + state_dir, + staging_root, + ] ) ensure_directories(cache_paths) - order = topological_order(COMPONENTS, DEPENDENCIES) + components = config.enabled_packages() + dependencies = { + name: config.packages[name].dependencies + for name in components + if config.packages[name].dependencies + } + order = topological_order(components, dependencies) component_inputs = { - component: component_input_digest(config_dir, component) for component in order + component: component_input_digest(config, component) for component in order } keys = desired_build_keys( order, - DEPENDENCIES, + dependencies, component_inputs, distro=args.distro, architecture=architecture, builder_image_id=builder_image_id, - build_script_digest=file_digest(generic_script), + build_scripts_digest=files_digest((root / "buildscripts").glob("*")), ) runner = podman_build_runner( root=root, @@ -361,6 +424,7 @@ def main(argv: Sequence[str] | None = None) -> int: ) rebuilt = rebuild_outdated( order, + config, keys, artifact_dir, state_dir, @@ -368,7 +432,6 @@ def main(argv: Sequence[str] | None = None) -> int: runner, force=args.force, ) - missing = [ component for component in order @@ -376,11 +439,7 @@ def main(argv: Sequence[str] | None = None) -> int: ] if missing: raise RuntimeError("packages are not current: " + ", ".join(missing)) - - if rebuilt: - print("Rebuilt packages: " + ", ".join(rebuilt)) - else: - print("All packages are up to date") + print("Rebuilt packages: " + ", ".join(rebuilt) if rebuilt else "All packages are up to date") return 0 diff --git a/contrib/ci/jobs/1-build-head/build.sh b/contrib/ci/jobs/1-build-head/build.sh @@ -1,13 +0,0 @@ -#!/bin/bash -set -exuo pipefail - -./contrib/ci/jobs/1-build-head/update-tags.sh - -./sandcastle-build - -mkdir -p /artifacts/sandcastle-ng/${CI_COMMIT_REF} # Variable comes from CI environment -podman tag taler-base-all:latest taler-base-all-head:latest -podman tag taler-base-all-head:latest taler-base-all-head:${CI_COMMIT_REF} -podman save \ - -o /artifacts/sandcastle-ng/${CI_COMMIT_REF}/taler-base-all-head.tar \ - taler-base-all-head:latest diff --git a/contrib/ci/jobs/1-build-head/job.sh b/contrib/ci/jobs/1-build-head/job.sh @@ -1,6 +0,0 @@ -#!/bin/bash -set -exuo pipefail - -job_dir=$(dirname "${BASH_SOURCE[0]}") - -"${job_dir}"/build.sh diff --git a/contrib/ci/jobs/1-build-head/update-tags.sh b/contrib/ci/jobs/1-build-head/update-tags.sh @@ -1,46 +0,0 @@ -#!/bin/bash -set -ex - -fetch_head() { - git ls-remote -q -h "${1}" master | cut -f1 -} - -GNUNET_HEAD=$(fetch_head "git://git.gnunet.org/gnunet") -EXCHANGE_HEAD=$(fetch_head "git://git.taler.net/exchange") -MERCHANT_HEAD=$(fetch_head "git://git.taler.net/merchant") -LIBEUFIN_HEAD=$(fetch_head "git://git.taler.net/libeufin") -MERCHANT_DEMOS_HEAD=$(fetch_head "git://git.taler.net/taler-merchant-demos") -WALLET_HEAD=$(fetch_head "git://git.taler.net/wallet-core") -SYNC_HEAD=$(fetch_head "git://git.taler.net/sync") -DONAU_HEAD=$(fetch_head "git://git.taler.net/donau") -MAILBOX_HEAD=$(fetch_head "git://git.taler.net/taler-mailbox") -TALDIR_HEAD=$(fetch_head "git://git.taler.net/taldir") -CHALLENGER_HEAD=$(fetch_head "git://git.taler.net/challenger") -TURNSTILE_HEAD=$(fetch_head "git://git.taler.net/turnstile") - -echo "master" > buildconfig/gnunet.tag -echo "master" > buildconfig/exchange.tag -echo "master" > buildconfig/merchant.tag -#echo "master" > buildconfig/libeufin.tag -echo "master" > buildconfig/merchant-demos.tag -echo "master" > buildconfig/wallet.tag -echo "master" > buildconfig/sync.tag -echo "master" > buildconfig/donau.tag -echo "master" > buildconfig/taler-directory.tag -echo "master" > buildconfig/taler-mailbox.tag -echo "master" > buildconfig/challenger.tag -echo "master" > buildconfig/turnstile.tag - -rm -f buildconfig/*.checkout -echo $GNUNET_HEAD > buildconfig/gnunet.checkout -echo $EXCHANGE_HEAD > buildconfig/exchange.checkout -echo $MERCHANT_HEAD > buildconfig/merchant.checkout -echo $LIBEUFIN_HEAD > buildconfig/libeufin.checkout -echo $MERCHANT_DEMOS_HEAD > buildconfig/merchant-demos.checkout -echo $WALLET_HEAD > buildconfig/wallet.checkout -echo $SYNC_HEAD > buildconfig/sync.checkout -echo $MAILBOX_HEAD > buildconfig/taler-directory.checkout -echo $TALDIR_HEAD > buildconfig/taler-directory.checkout -echo $DONAU_HEAD > buildconfig/donau.checkout -echo $CHALLENGER_HEAD > buildconfig/challenger.checkout -echo $TURNSTILE_HEAD > buildconfig/turnstile.checkout diff --git a/contrib/ci/jobs/2-deploy-head/config.ini b/contrib/ci/jobs/2-deploy-head/config.ini @@ -1,6 +0,0 @@ -[build] -HALT_ON_FAILURE = True -WARN_ON_FAILURE = True -CONTAINER_BUILD = False -CONTAINER_NAME = docker.io/library/debian:bookworm-slim -CONTAINER_ARCH = amd64 diff --git a/contrib/ci/jobs/2-deploy-head/job.sh b/contrib/ci/jobs/2-deploy-head/job.sh @@ -1,26 +0,0 @@ -#!/bin/bash -set -exuo pipefail - -ARTIFACT_PATH="/artifacts/sandcastle-ng/${CI_COMMIT_REF}/*.tar" - -RSYNC_HOST="firefly.gnunet.org" -RSYNC_PORT=22 -RSYNC_PATH="incoming" -RSYNC_USER="taler-head" -RSYNC_DEST="${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}" - -# Useful to print for debug -ls -alh $(dirname ${ARTIFACT_PATH}) - -## We need ssh and rsync -apt-get update -yqq -apt-get install -yqq --no-install-recommends \ - ssh-client rsync - -# Send our container tarball to the host -rsync -vP \ - --port ${RSYNC_PORT} \ - ${ARTIFACT_PATH} ${RSYNC_DEST} || (rm -f ${ARTIFACT_PATH} && exit 1) - -# Make sure we cleanup -rm -f ${ARTIFACT_PATH} diff --git a/packages.toml b/packages.toml @@ -0,0 +1,180 @@ +# Package definitions for Sandcastle. +# This file is rewritten in canonical form by `sandcastle-upgrade`. + +[repositories."challenger"] +url = "git://git.taler.net/challenger" +builder = "generic" + +[repositories."donau"] +url = "git://git.taler.net/donau.git" +builder = "generic" + +[repositories."gnunet"] +url = "git://git.gnunet.org/gnunet.git" +builder = "generic" + +[repositories."libeufin"] +url = "git://git.taler.net/libeufin" +builder = "generic" + +[repositories."paivana"] +url = "git://git.taler.net/paivana" +builder = "generic" + +[repositories."sync"] +url = "git://git.taler.net/sync.git" +builder = "generic" + +[repositories."taler-directory"] +url = "git://git.gnunet.org/taldir" +builder = "generic" + +[repositories."taler-exchange"] +url = "git://git.taler.net/exchange.git" +builder = "generic" + +[repositories."taler-mailbox"] +url = "git://git.gnunet.org/taler-mailbox" +builder = "generic" + +[repositories."taler-merchant"] +url = "git://git.taler.net/merchant.git" +builder = "generic" + +[repositories."taler-merchant-demos"] +url = "git://git.taler.net/taler-merchant-demos" +builder = "generic" + +[repositories."turnstile"] +url = "git://git.taler.net/turnstile" +builder = "generic" + +[repositories."typescript-core"] +url = "git://git.taler.net/taler-typescript-core.git" +builder = "pnpm-workspace" + +[packages."challenger"] +repository = "challenger" +tag = "v1.6.3-dev.1" +debian_path = "" +dependencies = ["taler-exchange"] +enabled = true + +[packages."challenger-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.6" +debian_path = "packages/challenger-webui" +dependencies = [] +enabled = true + +[packages."donau"] +repository = "donau" +tag = "v1.6.7" +debian_path = "" +dependencies = ["gnunet", "taler-exchange"] +enabled = true + +[packages."gnunet"] +repository = "gnunet" +tag = "v0.28.1-dev.5" +debian_path = "" +dependencies = [] +enabled = true + +[packages."libeufin"] +repository = "libeufin" +tag = "v1.6.9-dev.1" +debian_path = "" +dependencies = [] +enabled = true + +[packages."libeufin-bank-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.6" +debian_path = "packages/libeufin-bank-webui" +dependencies = [] +enabled = true + +[packages."paivana-httpd"] +repository = "paivana" +tag = "v1.6.4" +debian_path = "" +dependencies = ["gnunet", "taler-exchange", "taler-merchant"] +enabled = true + +[packages."sync"] +repository = "sync" +tag = "v1.6.0-dev.1" +debian_path = "" +dependencies = ["gnunet", "taler-exchange", "taler-merchant"] +enabled = false + +[packages."taler-auditor-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.6" +debian_path = "packages/taler-auditor-webui" +dependencies = [] +enabled = true + +[packages."taler-directory"] +repository = "taler-directory" +tag = "v1.5.1" +debian_path = "" +dependencies = [] +enabled = true + +[packages."taler-exchange"] +repository = "taler-exchange" +tag = "v1.6.15-dev.1" +debian_path = "" +dependencies = ["gnunet"] +enabled = true + +[packages."taler-harness"] +repository = "typescript-core" +tag = "v1.6.33-dev.6" +debian_path = "packages/taler-harness" +dependencies = [] +enabled = true + +[packages."taler-mailbox"] +repository = "taler-mailbox" +tag = "v1.5.1" +debian_path = "" +dependencies = [] +enabled = true + +[packages."taler-merchant"] +repository = "taler-merchant" +tag = "v1.6.19" +debian_path = "" +dependencies = ["gnunet", "taler-exchange", "donau"] +enabled = true + +[packages."taler-merchant-demos"] +repository = "taler-merchant-demos" +tag = "v1.6.3" +debian_path = "" +dependencies = [] +enabled = true + +[packages."taler-merchant-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.6" +debian_path = "packages/taler-merchant-webui" +dependencies = [] +enabled = true + +[packages."taler-wallet-cli"] +repository = "typescript-core" +tag = "v1.6.33-dev.6" +debian_path = "packages/taler-wallet-cli" +dependencies = [] +enabled = true + +[packages."turnstile"] +repository = "turnstile" +tag = "v0.0.2" +debian_path = "" +dependencies = [] +enabled = false diff --git a/sandcastle-build b/sandcastle-build @@ -47,7 +47,22 @@ builder_image=localhost/taler-sandcastle-package-builder:latest cache_dir=$SCRIPT_DIR/cache mkdir -p \ "$cache_dir/debian-trixie/apt-archives/partial" \ - "$cache_dir/debian-trixie/apt-lists/partial" + "$cache_dir/debian-trixie/apt-lists/partial" \ + "$cache_dir/npm" + +read -r turnstile_url turnstile_tag < <( + python3 - "$SCRIPT_DIR" <<'PY' +import sys + +root = sys.argv[1] +sys.path.insert(0, f"{root}/buildscripts") +from package_config import load_config + +config = load_config(f"{root}/packages.toml") +package = config.packages["turnstile"] +print(config.repository_for(package).url, package.tag, sep="\t") +PY +) # The nofile ulimit is required to prevent fakeroot from becoming sluggish. build_args=( @@ -55,12 +70,13 @@ build_args=( --log-level=debug --ulimit=nofile=2048:2048 -f "$containerfile" + --volume "$cache_dir/debian-trixie/apt-archives:/var/cache/apt/archives:z" + --volume "$cache_dir/debian-trixie/apt-lists:/var/lib/apt/lists:z" + --volume "$cache_dir/npm:/root/.npm:z" ) echo "Building package-builder image with containerfile: $containerfile" "${build_args[@]}" "${no_cache_args[@]}" \ - --volume "$cache_dir/debian-trixie/apt-archives:/var/cache/apt/archives:z" \ - --volume "$cache_dir/debian-trixie/apt-lists:/var/lib/apt/lists:z" \ --target base-system \ --tag "$builder_image" \ . @@ -73,6 +89,8 @@ python3 buildscripts/sandcastle_build_packages.py \ echo "Building final sandcastle image" exec "${build_args[@]}" "${no_cache_args[@]}" \ --build-arg "SANDCASTLE_VERSION=$sandcastle_version" \ + --build-arg "TURNSTILE_GIT_URL=$turnstile_url" \ + --build-arg "TURNSTILE_TAG=$turnstile_tag" \ --target taler-final \ --tag taler-base-all \ . diff --git a/sandcastle-upgrade b/sandcastle-upgrade @@ -4,154 +4,147 @@ # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) # SPDX-License-Identifier: GPL-3.0-or-later -# Upgrade the git tags in buildconfig/ to the latest upstream tag. -# Works like the "upgrade" subcommand of taler-pkg. +"""Upgrade package tags in packages.toml to their latest upstream versions.""" import argparse import os import re import subprocess import sys +from dataclasses import replace from pathlib import Path -# Tag syntax variants supported by buildscripts/sandcastle-build-generic: -# v$maj.$min.$patch => release version -# v$maj.$min.$patch-dev.$n => dev version -# deb-v$maj.$min.$patch-$revision => release version with debian revision -# Debian revisions of dev versions are *not* supported +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR / "buildscripts")) + +from package_config import ConfigError, load_config, write_config # noqa: E402 + + tag_re_release = re.compile(r"v(\d+)\.(\d+)\.(\d+)") tag_re_dev = re.compile(r"v(\d+)\.(\d+)\.(\d+)-dev\.(\d+)") tag_re_deb = re.compile(r"deb-v(\d+)\.(\d+)\.(\d+)(?:-(\d+))?") def tag_sortkey(tag): - """Get a sort key for a tag, or None if the tag syntax isn't supported. - - Dev versions sort before the corresponding release version, debian - revisions sort after it. - """ - m = tag_re_release.fullmatch(tag) - if m: - return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, 0) - m = tag_re_dev.fullmatch(tag) - if m: - return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 0, int(m.group(4))) - m = tag_re_deb.fullmatch(tag) - if m: - rev = m.group(4) - return (int(m.group(1)), int(m.group(2)), int(m.group(3)), 1, int(rev or 0)) + match = tag_re_release.fullmatch(tag) + if match: + return (int(match.group(1)), int(match.group(2)), int(match.group(3)), 1, 0) + match = tag_re_dev.fullmatch(tag) + if match: + return ( + int(match.group(1)), + int(match.group(2)), + int(match.group(3)), + 0, + int(match.group(4)), + ) + match = tag_re_deb.fullmatch(tag) + if match: + revision = match.group(4) + return ( + int(match.group(1)), + int(match.group(2)), + int(match.group(3)), + 1, + int(revision or 0), + ) return None def list_remote_tags(url): - """Get all tags from the git repo""" - cmd = ["git", "ls-remote", "--exit-code", "--refs", "--tags", url] - result = subprocess.run(cmd, capture_output=True, text=True, check=True) + result = subprocess.run( + ["git", "ls-remote", "--exit-code", "--refs", "--tags", url], + capture_output=True, + text=True, + check=True, + ) tags = [] for line in result.stdout.strip().split("\n"): parts = line.split() - if len(parts) < 2: - continue - # refs/tags/v1.0.0 -> v1.0.0 - tags.append(parts[1].split("/")[-1]) + if len(parts) >= 2: + tags.append(parts[1].split("/")[-1]) return tags def latest_tag(tags, dev): - """Find the newest supported tag, only considering dev tags if dev is set""" best = None bestkey = None - # Iterate in sorted order, so that the result doesn't depend on - # the order in which the remote lists its refs. for tag in sorted(tags): if not dev and tag_re_dev.fullmatch(tag): continue key = tag_sortkey(tag) - if key is None: - continue - if bestkey is None or key > bestkey: + if key is not None and (bestkey is None or key > bestkey): best = tag bestkey = key return best -def upgrade(cfg): - """Upgrade tag files in buildconfig to the latest upstream tag""" - names = cfg.components - if not names: - names = sorted(p.stem for p in Path("buildconfig").glob("*.tag")) - # Multiple components can share a repo, only ask each remote once. +def upgrade(cfg, config_path): + config = load_config(config_path) + names = cfg.components or sorted(config.packages) + unknown = sorted(set(names) - set(config.packages)) + if unknown: + raise ConfigError(f"unknown package(s): {', '.join(unknown)}") + remote_tags = {} upgraded = [] + updated_packages = dict(config.packages) for name in names: - tag_file = Path("buildconfig") / f"{name}.tag" - url_file = Path("buildconfig") / f"{name}.giturl" - if not url_file.exists(): - print(f"[?] {name} has no giturl, skipping", file=sys.stderr) - continue - giturl = url_file.read_text().strip() + package = config.packages[name] + giturl = config.repository_for(package).url if giturl not in remote_tags: remote_tags[giturl] = list_remote_tags(giturl) latest = latest_tag(remote_tags[giturl], cfg.dev) if latest is None: print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr) continue - curr = None - if tag_file.exists(): - curr = tag_file.read_text().strip() - currkey = tag_sortkey(curr) - if currkey is None: - print( - f"[?] {name} tag {curr} has unsupported syntax, skipping", - file=sys.stderr, - ) - continue - latestkey = tag_sortkey(latest) - if currkey > latestkey: - # Happens when the tag file pins a dev version but only - # production tags are considered. - print(f" {name} {curr} (newer than latest {latest})") - continue - if currkey == latestkey: - print(f" {name} {curr} (up to date)") - continue - print(f"[!] {name} {curr or '(none)'} -> {latest}") + currkey = tag_sortkey(package.tag) + if currkey is None: + print( + f"[?] {name} tag {package.tag} has unsupported syntax, skipping", + file=sys.stderr, + ) + continue + latestkey = tag_sortkey(latest) + if currkey > latestkey: + print(f" {name} {package.tag} (newer than latest {latest})") + continue + if currkey == latestkey: + print(f" {name} {package.tag} (up to date)") + continue + print(f"[!] {name} {package.tag} -> {latest}") upgraded.append(name) if not cfg.dry: - tag_file.write_text(latest + "\n") + updated_packages[name] = replace(package, tag=latest) + if not upgraded: print("nothing to upgrade") elif cfg.dry: print("would upgrade:", " ".join(upgraded)) else: + write_config(config_path, replace(config, packages=updated_packages)) print("upgraded:", " ".join(upgraded)) def main(): parser = argparse.ArgumentParser( prog="sandcastle-upgrade", - description="Upgrade component tags to the latest upstream version.", + description="Upgrade package tags to the latest upstream version.", ) parser.add_argument( "components", nargs="*", - help="Components to upgrade (default: all components in buildconfig)", + help="Packages to upgrade (default: all packages in packages.toml)", ) - parser.add_argument( - "--dev", - help="Also consider dev tags, not just production tags", - action="store_true", - default=False, - ) - parser.add_argument("--dry", help="Dry run", action="store_true", default=False) - + parser.add_argument("--dev", action="store_true", help="Also consider dev tags") + parser.add_argument("--dry", action="store_true", help="Dry run") args = parser.parse_args() - # buildconfig/ is relative to this script, so the script can be - # run from any working directory. - os.chdir(os.path.dirname(os.path.realpath(__file__))) - - upgrade(args) + os.chdir(SCRIPT_DIR) + try: + upgrade(args, SCRIPT_DIR / "packages.toml") + except ConfigError as exc: + parser.error(str(exc)) if __name__ == "__main__": diff --git a/tests/test_package_config.py b/tests/test_package_config.py @@ -0,0 +1,87 @@ +# This file is in the public domain. + +import stat +import tempfile +import tomllib +import unittest +from dataclasses import replace +from pathlib import Path + +from buildscripts.package_config import ( + ConfigError, + load_config, + parse_config, + serialize_config, + write_config, +) + + +VALID_CONFIG = { + "repositories": { + "workspace": { + "url": "git://example.test/workspace.git", + "builder": "pnpm-workspace", + } + }, + "packages": { + "application": { + "repository": "workspace", + "tag": "v1.2.3", + "debian_path": "packages/application", + "dependencies": ["library"], + "enabled": True, + }, + "library": {"repository": "workspace", "tag": "v1.2.2"}, + }, +} + + +class PackageConfigTests(unittest.TestCase): + def test_defaults_match_packaging_ng(self): + config = parse_config(VALID_CONFIG) + library = config.packages["library"] + self.assertEqual("", library.debian_path) + self.assertEqual((), library.dependencies) + self.assertTrue(library.enabled) + self.assertEqual("pnpm-workspace", config.builder_for(library)) + + def test_rejects_unknown_fields_references_and_cycles(self): + with self.assertRaisesRegex(ConfigError, "unknown field"): + parse_config({**VALID_CONFIG, "surprise": {}}) + data = { + **VALID_CONFIG, + "packages": { + **VALID_CONFIG["packages"], + "library": { + **VALID_CONFIG["packages"]["library"], + "dependencies": ["application"], + }, + }, + } + with self.assertRaisesRegex(ConfigError, "application -> library -> application"): + parse_config(data) + + def test_serializer_round_trips_and_atomic_writer_preserves_mode(self): + config = parse_config(VALID_CONFIG) + serialized = serialize_config(config) + self.assertEqual(config, parse_config(tomllib.loads(serialized))) + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "packages.toml" + path.write_text("old\n", encoding="utf-8") + path.chmod(0o640) + write_config(path, config) + self.assertEqual(config, load_config(path)) + self.assertEqual(0o640, stat.S_IMODE(path.stat().st_mode)) + + def test_repository_change_is_represented_in_config(self): + config = parse_config(VALID_CONFIG) + repositories = dict(config.repositories) + repositories["workspace"] = replace( + repositories["workspace"], url="git://example.test/new.git" + ) + changed = replace(config, repositories=repositories) + self.assertNotEqual(serialize_config(config), serialize_config(changed)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sandcastle_build_packages.py b/tests/test_sandcastle_build_packages.py @@ -1,26 +1,54 @@ # This file is in the public domain. -import os import runpy import shutil import subprocess import tempfile import unittest +from dataclasses import replace from pathlib import Path from unittest.mock import patch +from buildscripts.package_config import parse_config from buildscripts.sandcastle_build_packages import ( - COMPONENTS, - DEPENDENCIES, + CACHE_MOUNTS, component_input_digest, component_is_current, desired_build_keys, + group_components, podman_build_runner, rebuild_outdated, topological_order, ) +def packaging_config(*, workspace=True): + repositories = { + "generic": {"url": "https://example.test/generic.git"}, + "workspace": { + "url": "https://example.test/workspace.git", + "builder": "pnpm-workspace" if workspace else "generic", + }, + } + packages = { + "a": { + "repository": "generic", + "tag": "v1.0.0", + }, + "b": { + "repository": "workspace", + "tag": "v2.0.0", + "debian_path": "packages/b", + }, + "c": { + "repository": "workspace", + "tag": "v2.0.0", + "debian_path": "packages/c", + }, + } + return parse_config({"repositories": repositories, "packages": packages}) + + class DebianVersionTests(unittest.TestCase): @classmethod def setUpClass(cls): @@ -36,8 +64,10 @@ class DebianVersionTests(unittest.TestCase): def test_debian_revision_and_codename_are_preserved(self): version = self.generic["get_tag_debver"]("deb-v1.2.3-4") - with_codename = self.generic["make_codename_version"](version, "trixie") - self.assertEqual("1.2.3-4+trixie", with_codename) + self.assertEqual( + "1.2.3-4+trixie", + self.generic["make_codename_version"](version, "trixie"), + ) class DependencyKeyTests(unittest.TestCase): @@ -53,7 +83,7 @@ class DependencyKeyTests(unittest.TestCase): distro="debian-trixie", architecture="amd64", builder_image_id="builder-1", - build_script_digest="script-1", + build_scripts_digest="scripts-1", ) def test_topological_order_puts_dependencies_first(self): @@ -62,86 +92,57 @@ class DependencyKeyTests(unittest.TestCase): self.assertLess(order.index("b"), order.index("c")) self.assertEqual(set(self.components), set(order)) - def test_changed_package_invalidates_only_it_and_reverse_dependencies(self): + def test_dependency_change_invalidates_reverse_dependencies(self): before = self.keys({"a": "a1", "b": "b1", "c": "c1", "d": "d1"}) after = self.keys({"a": "a2", "b": "b1", "c": "c1", "d": "d1"}) - changed = { - component for component in before if before[component] != after[component] - } + changed = {name for name in before if before[name] != after[name]} self.assertEqual({"a", "b", "c"}, changed) - def test_changed_consumer_does_not_invalidate_its_dependency(self): + def test_consumer_change_does_not_invalidate_dependency(self): before = self.keys({"a": "a1", "b": "b1", "c": "c1", "d": "d1"}) after = self.keys({"a": "a1", "b": "b2", "c": "c1", "d": "d1"}) - changed = { - component for component in before if before[component] != after[component] - } + changed = {name for name in before if before[name] != after[name]} self.assertEqual({"b", "c"}, changed) - def test_cycles_are_rejected(self): - with self.assertRaisesRegex(ValueError, "cycle"): - topological_order(("a", "b"), {"a": ("b",), "b": ("a",)}) - def test_exchange_change_rebuilds_its_real_reverse_dependencies(self): - order = topological_order(COMPONENTS, DEPENDENCIES) - inputs = {component: "unchanged" for component in COMPONENTS} - before = desired_build_keys( - order, - DEPENDENCIES, - inputs, - distro="debian-trixie", - architecture="amd64", - builder_image_id="builder-1", - build_script_digest="script-1", - ) - inputs["taler-exchange"] = "changed" - after = desired_build_keys( - order, - DEPENDENCIES, - inputs, - distro="debian-trixie", - architecture="amd64", - builder_image_id="builder-1", - build_script_digest="script-1", - ) - changed = { - component - for component in COMPONENTS - if before[component] != after[component] - } - self.assertEqual( - { - "taler-exchange", - "donau", - "taler-merchant", - "paivana-httpd", - "challenger", - }, - changed, +class ComponentInputDigestTests(unittest.TestCase): + def test_unrelated_package_change_does_not_change_digest(self): + config = packaging_config() + before = component_input_digest(config, "a") + packages = dict(config.packages) + packages["b"] = replace(packages["b"], tag="v9.0.0") + after = component_input_digest(replace(config, packages=packages), "a") + self.assertEqual(before, after) + + def test_repository_or_tag_change_changes_digest(self): + config = packaging_config() + before = component_input_digest(config, "a") + packages = dict(config.packages) + packages["a"] = replace(packages["a"], tag="v1.0.1") + self.assertNotEqual( + before, + component_input_digest(replace(config, packages=packages), "a"), ) -class ComponentInputDigestTests(unittest.TestCase): - def test_checkout_pin_changes_component_digest(self): - with tempfile.TemporaryDirectory() as temporary: - config_dir = Path(temporary) - (config_dir / "component.tag").write_text("master\n", encoding="utf-8") - (config_dir / "component.giturl").write_text( - "https://example.com/component.git\n", encoding="utf-8" - ) +class BuildGroupingTests(unittest.TestCase): + def test_workspace_packages_with_same_tag_share_group(self): + config = packaging_config() + groups = group_components(config, {"b", "c"}, ["a", "b", "c"]) + self.assertEqual([["b", "c"]], groups) - without_checkout = component_input_digest(config_dir, "component") - (config_dir / "component.checkout").write_text( - "first-commit\n", encoding="utf-8" - ) - first_checkout = component_input_digest(config_dir, "component") - (config_dir / "component.checkout").write_text( - "second-commit\n", encoding="utf-8" - ) - second_checkout = component_input_digest(config_dir, "component") + def test_different_tags_create_different_groups(self): + config = packaging_config() + packages = dict(config.packages) + packages["c"] = replace(packages["c"], tag="v2.0.1") + config = replace(config, packages=packages) + groups = group_components(config, {"b", "c"}, ["a", "b", "c"]) + self.assertEqual({("b",), ("c",)}, {tuple(group) for group in groups}) - self.assertNotEqual(without_checkout, first_checkout) - self.assertNotEqual(first_checkout, second_checkout) + def test_generic_packages_remain_isolated(self): + config = packaging_config(workspace=False) + groups = group_components(config, {"b", "c"}, ["a", "b", "c"]) + self.assertEqual([["b"], ["c"]], groups) class ArtifactStateTests(unittest.TestCase): @@ -158,66 +159,75 @@ class ArtifactStateTests(unittest.TestCase): def tearDown(self): self.temporary.cleanup() - def record_old_build(self): - component_dir = self.artifacts / "a" + def record_old_build(self, component): + component_dir = self.artifacts / component component_dir.mkdir() (component_dir / "old.deb").write_bytes(b"old") - (self.state / "a.key").write_text("old-key\n", encoding="utf-8") + (self.state / f"{component}.key").write_text("old-key\n", encoding="utf-8") - def test_failed_build_keeps_previous_artifacts_and_key(self): - self.record_old_build() + def test_failed_group_keeps_every_previous_artifact_and_key(self): + config = packaging_config() + for component in ("b", "c"): + self.record_old_build(component) - def fail_build(_component, staging_dir): - (staging_dir / "new.deb").write_bytes(b"new") + def fail_build(components, staging_dir): + for component in components: + (staging_dir / component / "new.deb").write_bytes(b"new") raise RuntimeError("build failed") with self.assertRaisesRegex(RuntimeError, "build failed"): rebuild_outdated( - ["a"], - {"a": "new-key"}, + ["b", "c"], + config, + {"b": "new-b", "c": "new-c"}, self.artifacts, self.state, self.staging, fail_build, ) - self.assertEqual(b"old", (self.artifacts / "a" / "old.deb").read_bytes()) - self.assertEqual("old-key", (self.state / "a.key").read_text().strip()) - self.assertFalse( - component_is_current(self.artifacts, self.state, "a", "new-key") - ) + for component in ("b", "c"): + self.assertEqual( + b"old", (self.artifacts / component / "old.deb").read_bytes() + ) + self.assertEqual( + "old-key", (self.state / f"{component}.key").read_text().strip() + ) self.assertEqual([], list(self.staging.iterdir())) - def test_successful_build_promotes_artifacts_and_key(self): - self.record_old_build() + def test_successful_group_promotes_each_package(self): + config = packaging_config() - def succeed_build(_component, staging_dir): - (staging_dir / "new.deb").write_bytes(b"new") + def succeed_build(components, staging_dir): + for component in components: + (staging_dir / component / f"{component}.deb").write_bytes( + component.encode() + ) rebuilt = rebuild_outdated( - ["a"], - {"a": "new-key"}, + ["b", "c"], + config, + {"b": "key-b", "c": "key-c"}, self.artifacts, self.state, self.staging, succeed_build, ) - - self.assertEqual(["a"], rebuilt) - self.assertFalse((self.artifacts / "a" / "old.deb").exists()) - self.assertEqual(b"new", (self.artifacts / "a" / "new.deb").read_bytes()) - self.assertTrue( - component_is_current(self.artifacts, self.state, "a", "new-key") - ) - - def test_matching_key_without_an_artifact_is_not_current(self): - (self.state / "a.key").write_text("key\n", encoding="utf-8") - self.assertFalse(component_is_current(self.artifacts, self.state, "a", "key")) + self.assertEqual(["b", "c"], rebuilt) + for component in ("b", "c"): + self.assertTrue( + component_is_current( + self.artifacts, + self.state, + component, + f"key-{component}", + ) + ) class PackageRunnerTests(unittest.TestCase): @patch("buildscripts.sandcastle_build_packages.subprocess.run") - def test_package_container_has_bounded_nofile_limit(self, run): + def test_package_container_has_all_cache_mounts_and_group(self, run): with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) runner = podman_build_runner( @@ -229,16 +239,19 @@ class PackageRunnerTests(unittest.TestCase): artifact_dir=root / "artifacts", cache_dir=root / "cache", ) - - runner("component", root / "staging") + runner(["b", "c"], root / "staging") command = run.call_args.args[0] self.assertIn("--ulimit=nofile=2048:2048", command) + self.assertEqual(["b", "c"], command[-2:]) + for _source, target in CACHE_MOUNTS: + self.assertTrue(any(f"target={target}" in argument for argument in command)) + self.assertTrue(any("target=/var/cache/apt/archives" in arg for arg in command)) run.assert_called_once_with(command, check=True) class BuildOrchestrationTests(unittest.TestCase): - def test_build_phases_and_no_cache_propagation(self): + def test_build_phases_cache_mounts_and_no_cache_propagation(self): repository = Path(__file__).resolve().parents[1] with tempfile.TemporaryDirectory() as temporary: root = Path(temporary) @@ -246,21 +259,29 @@ class BuildOrchestrationTests(unittest.TestCase): (root / "buildscripts").mkdir() shutil.copy(repository / "sandcastle-build", root / "sandcastle-build") (root / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8") + (root / "packages.toml").touch() + (root / "buildscripts" / "package_config.py").touch() (root / "buildscripts" / "sandcastle_build_packages.py").touch() log = root / "calls" - fake_command = '#!/bin/sh\nprintf \'%s\' "$0" >> "$CALL_LOG"\nprintf \' <%s>\' "$@" >> "$CALL_LOG"\nprintf \'\\n\' >> "$CALL_LOG"\n' + fake_command = """#!/bin/sh +if [ \"$1\" = \"-\" ]; then + printf 'git://example.test/turnstile\\tv1.0.0\\n' + exit 0 +fi +printf '%s' \"$0\" >> \"$CALL_LOG\" +printf ' <%s>' \"$@\" >> \"$CALL_LOG\" +printf '\\n' >> \"$CALL_LOG\" +""" for command in ("podman", "python3"): executable = root / "bin" / command executable.write_text(fake_command, encoding="utf-8") executable.chmod(0o755) git = root / "bin" / "git" - git.write_text( - "#!/bin/sh\nprintf 'sandcastle-version\\n'\n", encoding="utf-8" - ) + git.write_text("#!/bin/sh\nprintf 'sandcastle-version\\n'\n", encoding="utf-8") git.chmod(0o755) - environment = os.environ.copy() + environment = dict(**__import__("os").environ) environment["PATH"] = f"{root / 'bin'}:{environment['PATH']}" environment["CALL_LOG"] = str(log) subprocess.run( @@ -271,33 +292,17 @@ class BuildOrchestrationTests(unittest.TestCase): stdout=subprocess.PIPE, text=True, ) - calls = log.read_text(encoding="utf-8").splitlines() self.assertEqual(3, len(calls)) self.assertIn("<--target> <base-system>", calls[0]) - self.assertIn("<--no-cache>", calls[0]) - self.assertIn("<buildscripts/sandcastle_build_packages.py>", calls[1]) self.assertIn("<--force>", calls[1]) self.assertIn("<--target> <taler-final>", calls[2]) - self.assertIn( - "<--build-arg> <SANDCASTLE_VERSION=sandcastle-version>", calls[2] - ) + self.assertIn("<--build-arg> <TURNSTILE_TAG=v1.0.0>", calls[2]) + self.assertIn("<--no-cache>", calls[0]) self.assertIn("<--no-cache>", calls[2]) - - log.write_text("", encoding="utf-8") - subprocess.run( - [str(root / "sandcastle-build")], - check=True, - cwd=root, - env=environment, - stdout=subprocess.PIPE, - text=True, - ) - calls = log.read_text(encoding="utf-8").splitlines() - self.assertEqual(3, len(calls)) - self.assertNotIn("<--no-cache>", calls[0]) - self.assertNotIn("<--force>", calls[1]) - self.assertNotIn("<--no-cache>", calls[2]) + for call in (calls[0], calls[2]): + self.assertIn(":/var/cache/apt/archives:z>", call) + self.assertIn(":/root/.npm:z>", call) if __name__ == "__main__": diff --git a/tests/test_sandcastle_upgrade.py b/tests/test_sandcastle_upgrade.py @@ -0,0 +1,59 @@ +# This file is in the public domain. + +import importlib.machinery +import importlib.util +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from buildscripts.package_config import load_config, parse_config, write_config + + +ROOT = Path(__file__).resolve().parents[1] +LOADER = importlib.machinery.SourceFileLoader( + "sandcastle_upgrade", str(ROOT / "sandcastle-upgrade") +) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +UPGRADE = importlib.util.module_from_spec(SPEC) +LOADER.exec_module(UPGRADE) + + +class UpgradeTests(unittest.TestCase): + def test_shared_repository_is_queried_once_and_tags_are_written(self): + config = parse_config( + { + "repositories": { + "workspace": {"url": "git://example.test/workspace.git"} + }, + "packages": { + "one": {"repository": "workspace", "tag": "v1.0.0"}, + "two": {"repository": "workspace", "tag": "v1.0.0"}, + }, + } + ) + with tempfile.TemporaryDirectory() as temporary: + config_path = Path(temporary) / "packages.toml" + write_config(config_path, config) + arguments = SimpleNamespace(components=["one", "two"], dev=False, dry=False) + with patch.object( + UPGRADE, "list_remote_tags", return_value=["v1.0.0", "v1.0.1"] + ) as list_remote_tags: + UPGRADE.upgrade(arguments, config_path) + + updated = load_config(config_path) + self.assertEqual("v1.0.1", updated.packages["one"].tag) + self.assertEqual("v1.0.1", updated.packages["two"].tag) + list_remote_tags.assert_called_once_with("git://example.test/workspace.git") + + def test_unknown_package_is_rejected(self): + with self.assertRaisesRegex(UPGRADE.ConfigError, "unknown package"): + UPGRADE.upgrade( + SimpleNamespace(components=["missing"], dev=False, dry=True), + ROOT / "packages.toml", + ) + + +if __name__ == "__main__": + unittest.main()