commit b216cf3edee8baa7d584c8bfd65f2a81d01c96c4
parent 26b92a100087858ac5a971dfaa155cbe6813765c
Author: Florian Dold <dold@taler.net>
Date: Sat, 22 Aug 2026 15:04:33 +0200
sandcastle: build component packages incrementally
We now use a similar approach as taler-pkg from packaging/ng.
Diffstat:
8 files changed, 803 insertions(+), 127 deletions(-)
diff --git a/.containerignore b/.containerignore
@@ -0,0 +1,7 @@
+.git/
+cache/
+credentials/
+exported/
+packages/.state/
+packages/.staging/
+**/__pycache__/
diff --git a/.gitignore b/.gitignore
@@ -1,2 +1,5 @@
/credentials/*
/exported/*
+/cache/
+/packages/
+__pycache__/
diff --git a/Dockerfile b/Dockerfile
@@ -72,82 +72,9 @@ RUN DEBIAN_FRONTEND=noninteractive \
equivs \
;
-# FIXME: Try to use debian packages where possible
+# FIXME: Try to use Debian packages where possible.
RUN npm install -g pnpm@10
-COPY buildscripts/sandcastle-build-generic /bin/
-
-# GNUnet
-FROM base-system AS gnunet
-COPY buildconfig/gnunet.* /buildconfig/
-RUN sandcastle-build-generic gnunet
-
-# Directory
-FROM base-system AS taler-directory
-COPY buildconfig/taler-directory.* /buildconfig/
-RUN sandcastle-build-generic taler-directory
-
-# Mailbox
-FROM base-system AS taler-mailbox
-COPY buildconfig/taler-mailbox.* /buildconfig/
-RUN sandcastle-build-generic taler-mailbox
-
-# Exchange
-FROM gnunet as taler-exchange
-COPY buildconfig/taler-exchange.* /buildconfig/
-RUN sandcastle-build-generic taler-exchange
-
-# Auditor Webui
-FROM base-system as taler-auditor-webui
-COPY buildconfig/taler-auditor-webui.* /buildconfig/
-RUN sandcastle-build-generic taler-auditor-webui
-
-# Donau
-FROM taler-exchange as donau
-COPY buildconfig/donau.* /buildconfig/
-RUN sandcastle-build-generic donau
-
-# Merchant
-FROM donau as taler-merchant
-COPY buildconfig/taler-merchant.* /buildconfig/
-RUN sandcastle-build-generic taler-merchant
-
-# Paivana (needs libtalermerchant from the merchant)
-FROM taler-merchant as paivana-httpd
-COPY buildconfig/paivana-httpd.* /buildconfig/
-RUN sandcastle-build-generic paivana-httpd
-
-# Challenger
-FROM base-system as challenger-webui
-COPY buildconfig/challenger-webui.* /buildconfig/
-RUN sandcastle-build-generic challenger-webui
-
-FROM taler-exchange as challenger
-COPY buildconfig/challenger.* /buildconfig/
-RUN sandcastle-build-generic challenger
-
-# Libeufin
-FROM base-system as libeufin
-COPY buildconfig/libeufin.* /buildconfig/
-RUN sandcastle-build-generic libeufin
-
-# Merchant demos
-FROM base-system as taler-merchant-demos
-COPY buildconfig/taler-merchant-demos.* /buildconfig/
-RUN sandcastle-build-generic taler-merchant-demos
-
-FROM base-system as taler-wallet-cli
-COPY buildconfig/taler-wallet-cli.* /buildconfig/
-RUN sandcastle-build-generic taler-wallet-cli
-
-FROM base-system as taler-harness
-COPY buildconfig/taler-harness.* /buildconfig/
-RUN sandcastle-build-generic taler-harness
-
-FROM base-system as taler-merchant-webui
-COPY buildconfig/taler-merchant-webui.* /buildconfig/
-RUN sandcastle-build-generic taler-merchant-webui
-
FROM base-system as turnstile
COPY buildconfig/turnstile.* /buildconfig/
RUN TAG=$(cat /buildconfig/turnstile.tag) && \
@@ -184,25 +111,13 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -y upgrade && apt-g
php-curl \
;
-RUN mkdir -p /packages
-COPY --from=gnunet /packages/gnunet/* /packages/
-COPY --from=taler-directory /packages/taler-directory/* /packages/
-COPY --from=taler-mailbox /packages/taler-mailbox/* /packages/
-COPY --from=taler-exchange /packages/taler-exchange/* /packages/
-COPY --from=taler-merchant /packages/taler-merchant/* /packages/
-COPY --from=taler-wallet-cli /packages/taler-wallet-cli/* /packages/
-COPY --from=taler-harness /packages/taler-harness/* /packages/
-COPY --from=taler-merchant-webui /packages/taler-merchant-webui/* /packages/
-COPY --from=taler-auditor-webui /packages/taler-auditor-webui/* /packages/
-COPY --from=libeufin /packages/libeufin/* /packages/
-COPY --from=taler-merchant-demos /packages/taler-merchant-demos/* /packages/
-COPY --from=challenger-webui /packages/challenger-webui/* /packages/
-COPY --from=challenger /packages/challenger/* /packages/
-COPY --from=donau /packages/donau/* /packages/
-COPY --from=paivana-httpd /packages/paivana-httpd/* /packages/
+COPY packages/debian-trixie/ /packages/
COPY --from=turnstile /opt/turnstile /opt/turnstile
RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -y upgrade && \
- apt-get install --no-install-recommends -y /packages/*.deb
+ package_dir="/packages/$(dpkg --print-architecture)" && \
+ test -d "$package_dir" && \
+ find "$package_dir" -type f -name '*.deb' -print0 | \
+ xargs -0 apt-get install --no-install-recommends -y
COPY systemd/setup-sandcastle.service /etc/systemd/system/
RUN systemctl enable setup-sandcastle.service
# Disable potentially problem-causing services
diff --git a/README.md b/README.md
@@ -59,6 +59,16 @@ use `./print-latest-versions`.
2. Run `./sandcastle-build` to build the Taler container. The resulting container
will be tagged as `taler-base-all`.
+ Package builds are incremental. Built Debian packages and their build keys
+ are kept below `packages/`, while downloaded build data is kept below
+ `cache/`. A component is rebuilt when its build configuration, package
+ builder, generic build logic, or one of its package dependencies changes.
+ Unrelated packages remain untouched. Failed builds are not recorded as
+ current, so they are retried on the next invocation.
+
+ Use `./sandcastle-build --no-cache` to rebuild the package builder, every
+ component package, and the final image without cached image layers.
+
By default `sandcastle-build` builds from the `Dockerfile`
in the same directory as the script.
Anyhow you can pass a different container file like this:
diff --git a/buildscripts/sandcastle-build-generic b/buildscripts/sandcastle-build-generic
@@ -10,12 +10,11 @@
import glob
import os
-import shlex
import shutil
import subprocess
import sys
from email.utils import formatdate
-from logging import log, warning
+from logging import warning
def run_cmd(cmd, shell=False, cwd=None, env=None):
@@ -39,27 +38,39 @@ def get_tag_debver(tag):
if d < 0:
return tag[1:]
else:
- return tag[1:d] + "~dev" + tag[d + len(devsuff)]
+ return tag[1:d] + "~dev" + tag[d + len(devsuff) :]
if tag.startswith("deb-v"):
tag = tag[5:]
if "-" in tag:
- a, b = tag.split("-")
- return a + ":" + b
+ upstream, revision = tag.split("-", 1)
+ return upstream + "-" + revision
return tag
warning(f"unexpected tag format: {tag}")
return "100.0.0" # Version big enough to work every time
+def make_codename_version(deb_version, codename):
+ """Make locally-built packages distinct from other distro builds."""
+ if "-" in deb_version:
+ return f"{deb_version}+{codename}"
+ return f"{deb_version}-0+{codename}"
+
+
def main():
if "LD_LIBRARY_PATH" in os.environ:
del os.environ["LD_LIBRARY_PATH"]
# Arguments
- if len(sys.argv) < 2:
- print(f"Usage: {sys.argv[0]} <PACKAGE>", file=sys.stderr)
+ if len(sys.argv) != 4:
+ print(
+ f"Usage: {sys.argv[0]} <PACKAGE> <CODENAME> <ARCH>",
+ file=sys.stderr,
+ )
sys.exit(1)
package = sys.argv[1]
+ codename = sys.argv[2]
+ architecture = sys.argv[3]
# Path of the debian/ folder in the repository
DEBIANPATH = ""
@@ -68,22 +79,31 @@ def main():
with open(debpath_file, "r") as f:
DEBIANPATH = f.read().strip()
- print(f"Building {package} with generic build logic", file=sys.stderr)
+ print(
+ f"Building {package} for {codename}/{architecture} with generic build logic",
+ file=sys.stderr,
+ )
with open(f"/buildconfig/{package}.tag", "r") as f:
tag = f.read().strip()
- outdir = f"/packages/{package}/"
-
+ outdir = "/out"
os.makedirs(outdir, exist_ok=True)
- os.chdir("/packages/")
+ os.chdir("/pkgdir")
- # Using shell=True here to easily handle the pipe logic
- run_cmd("dpkg-scanpackages . | xz - > /packages/Packages.xz", shell=True)
+ run_cmd(
+ [
+ "bash",
+ "-o",
+ "pipefail",
+ "-c",
+ "dpkg-scanpackages . /dev/null | xz - > Packages.xz",
+ ]
+ )
with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as f:
- f.write("deb [trusted=yes] file:/packages ./\n")
+ f.write("deb [trusted=yes] file:/pkgdir ./\n")
run_cmd(["apt-get", "update"])
@@ -101,7 +121,7 @@ def main():
build_pkg_path = os.path.join("/build", package, DEBIANPATH)
os.chdir(build_pkg_path)
- deb_version = get_tag_debver(tag)
+ deb_version = make_codename_version(get_tag_debver(tag), codename)
# Bootstrap and Install Deps
os.chdir(os.path.join("/build", package))
@@ -142,16 +162,13 @@ def main():
# Build Package
run_cmd(["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"])
- # Copy artifacts
- # Globs for ../*.deb and ../*.ddeb relative to current dir
+ # Copy binary packages into the host-side staging directory. Debug packages
+ # are intentionally omitted from the final sandcastle image.
deb_files = glob.glob("../*.deb")
- ddeb_files = glob.glob("../*.ddeb")
-
- for f in deb_files:
- shutil.copy(f, outdir)
-
- for f in ddeb_files:
- shutil.copy(f, outdir)
+ if not deb_files:
+ raise RuntimeError(f"build of {package} produced no .deb packages")
+ for deb_file in deb_files:
+ shutil.copy(deb_file, outdir)
if __name__ == "__main__":
diff --git a/buildscripts/sandcastle_build_packages.py b/buildscripts/sandcastle_build_packages.py
@@ -0,0 +1,387 @@
+#!/usr/bin/env python3
+# This file is in the public domain.
+"""Build sandcastle Debian packages incrementally in fresh containers."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import os
+import shutil
+import subprocess
+import sys
+import uuid
+from collections.abc import Callable, Iterable, Mapping, Sequence
+from pathlib import Path
+
+
+DISTRO = "debian-trixie"
+BUILDER_IMAGE = "localhost/taler-sandcastle-package-builder:latest"
+
+# Keep this list in sync with the packages installed by the final image.
+COMPONENTS = (
+ "gnunet",
+ "taler-directory",
+ "taler-mailbox",
+ "taler-exchange",
+ "taler-auditor-webui",
+ "donau",
+ "taler-merchant",
+ "paivana-httpd",
+ "challenger-webui",
+ "challenger",
+ "libeufin",
+ "taler-merchant-demos",
+ "taler-wallet-cli",
+ "taler-harness",
+ "taler-merchant-webui",
+)
+
+DEPENDENCIES = {
+ "taler-exchange": ("gnunet",),
+ "donau": ("gnunet", "taler-exchange"),
+ "taler-merchant": ("gnunet", "taler-exchange", "donau"),
+ "challenger": ("taler-exchange",),
+ "paivana-httpd": ("gnunet", "taler-exchange", "taler-merchant"),
+}
+
+BuildRunner = Callable[[str, Path], None]
+
+
+def topological_order(
+ components: Sequence[str], dependencies: Mapping[str, Sequence[str]]
+) -> list[str]:
+ """Return components with every dependency before its consumers."""
+ component_set = set(components)
+ if len(component_set) != len(components):
+ raise ValueError("component list contains duplicates")
+
+ order: list[str] = []
+ permanent: set[str] = set()
+ temporary: set[str] = set()
+
+ def visit(component: str) -> None:
+ if component in permanent:
+ return
+ if component in temporary:
+ raise ValueError(f"dependency cycle involving {component}")
+ if component not in component_set:
+ raise ValueError(f"unknown component in dependency graph: {component}")
+
+ temporary.add(component)
+ for dependency in dependencies.get(component, ()):
+ visit(dependency)
+ temporary.remove(component)
+ permanent.add(component)
+ order.append(component)
+
+ for component in components:
+ visit(component)
+ return order
+
+
+def file_digest(path: Path) -> str:
+ digest = hashlib.sha256()
+ with path.open("rb") as source:
+ for chunk in iter(lambda: source.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def component_input_digest(config_dir: Path, component: str) -> str:
+ """Hash all build configuration that belongs to one component."""
+ inputs: dict[str, str | None] = {}
+ required_suffixes = ("tag", "giturl")
+ optional_suffixes = ("debpath", "checkout")
+ for suffix in required_suffixes + optional_suffixes:
+ path = config_dir / f"{component}.{suffix}"
+ if suffix in required_suffixes and not path.is_file():
+ raise FileNotFoundError(f"missing build configuration: {path}")
+ inputs[suffix] = file_digest(path) if path.is_file() else None
+ return hashlib.sha256(canonical_json(inputs)).hexdigest()
+
+
+def canonical_json(value: object) -> bytes:
+ return json.dumps(value, sort_keys=True, separators=(",", ":")).encode()
+
+
+def desired_build_keys(
+ order: Sequence[str],
+ dependencies: Mapping[str, Sequence[str]],
+ component_inputs: Mapping[str, str],
+ *,
+ distro: str,
+ architecture: str,
+ builder_image_id: str,
+ build_script_digest: str,
+) -> dict[str, str]:
+ """Compute recursive keys so dependency changes invalidate consumers."""
+ keys: dict[str, str] = {}
+ for component in order:
+ payload = {
+ "schema": 1,
+ "component": component,
+ "component_input": component_inputs[component],
+ "distro": distro,
+ "architecture": architecture,
+ "builder_image_id": builder_image_id,
+ "build_script": build_script_digest,
+ "dependencies": {
+ dependency: keys[dependency]
+ for dependency in sorted(dependencies.get(component, ()))
+ },
+ }
+ keys[component] = hashlib.sha256(canonical_json(payload)).hexdigest()
+ return keys
+
+
+def state_path(state_dir: Path, component: str) -> Path:
+ return state_dir / f"{component}.key"
+
+
+def package_files(artifact_dir: Path, component: str) -> list[Path]:
+ return sorted((artifact_dir / component).glob("*.deb"))
+
+
+def component_is_current(
+ artifact_dir: Path, state_dir: Path, component: str, desired_key: str
+) -> bool:
+ marker = state_path(state_dir, component)
+ try:
+ recorded_key = marker.read_text(encoding="utf-8").strip()
+ except FileNotFoundError:
+ return False
+ return recorded_key == desired_key and bool(package_files(artifact_dir, component))
+
+
+def write_state_atomically(path: Path, desired_key: str) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
+ temporary.write_text(f"{desired_key}\n", encoding="utf-8")
+ os.replace(temporary, path)
+
+
+def promote_artifacts(staging_dir: Path, artifact_dir: Path, component: str) -> None:
+ """Replace a component's artifacts while retaining the old set on failure."""
+ destination = artifact_dir / component
+ artifact_dir.mkdir(parents=True, exist_ok=True)
+ backup = staging_dir.with_name(f".{component}.{uuid.uuid4().hex}.old")
+
+ if destination.exists():
+ os.replace(destination, backup)
+ try:
+ os.replace(staging_dir, destination)
+ except BaseException:
+ if backup.exists():
+ os.replace(backup, destination)
+ raise
+ if backup.exists():
+ shutil.rmtree(backup)
+
+
+def rebuild_outdated(
+ order: Sequence[str],
+ desired_keys: Mapping[str, str],
+ artifact_dir: Path,
+ state_dir: Path,
+ staging_root: Path,
+ run_build: BuildRunner,
+ *,
+ force: bool = False,
+) -> list[str]:
+ """Build outdated components in order and promote only successful outputs."""
+ rebuilt: list[str] = []
+ for component in order:
+ desired_key = desired_keys[component]
+ if not force and component_is_current(
+ artifact_dir, state_dir, component, desired_key
+ ):
+ print(f"Package {component}: up to date")
+ continue
+
+ print(f"Package {component}: rebuilding")
+ staging_dir = staging_root / f"{component}.{uuid.uuid4().hex}"
+ staging_dir.mkdir(parents=True, exist_ok=False)
+ try:
+ run_build(component, staging_dir)
+ outputs = sorted(staging_dir.glob("*.deb"))
+ if not outputs:
+ raise RuntimeError(f"build of {component} produced no .deb packages")
+ promote_artifacts(staging_dir, artifact_dir, component)
+ write_state_atomically(state_path(state_dir, component), desired_key)
+ rebuilt.append(component)
+ finally:
+ if staging_dir.exists():
+ shutil.rmtree(staging_dir)
+ return rebuilt
+
+
+def inspect_builder(image: str) -> tuple[str, str]:
+ result = subprocess.run(
+ ["podman", "image", "inspect", "--format", "{{.Id}} {{.Architecture}}", image],
+ check=True,
+ stdout=subprocess.PIPE,
+ text=True,
+ )
+ fields = result.stdout.strip().split()
+ if len(fields) != 2:
+ raise RuntimeError(f"could not determine identity and architecture of {image}")
+ return fields[0], fields[1]
+
+
+def ensure_directories(paths: Iterable[Path]) -> None:
+ for path in paths:
+ path.mkdir(parents=True, exist_ok=True)
+
+
+def podman_build_runner(
+ *,
+ root: Path,
+ image: str,
+ architecture: str,
+ distro: str,
+ codename: str,
+ artifact_dir: Path,
+ cache_dir: Path,
+) -> BuildRunner:
+ buildscripts_dir = root / "buildscripts"
+ buildconfig_dir = root / "buildconfig"
+ mounts = (
+ (cache_dir / "gradle", "/root/.gradle/caches"),
+ (cache_dir / "pnpm", "/root/.local/share/pnpm/store"),
+ (cache_dir / "cargo-registry", "/root/.cargo/registry"),
+ (cache_dir / "cargo-git", "/root/.cargo/git"),
+ (cache_dir / "cargo-build", "/root/.cargo-build"),
+ (cache_dir / distro / "apt-archives", "/var/cache/apt/archives"),
+ (cache_dir / distro / "apt-lists", "/var/lib/apt/lists"),
+ )
+
+ def run_build(component: str, staging_dir: Path) -> None:
+ command = [
+ "podman",
+ "run",
+ "--rm",
+ "--arch",
+ architecture,
+ "--ulimit=nofile=2048:2048",
+ "--entrypoint=/bin/python3",
+ "--security-opt",
+ "label=disable",
+ ]
+ for source, target in mounts:
+ command.extend(["--mount", f"type=bind,source={source},target={target}"])
+ command.extend(
+ [
+ "--env",
+ "CARGO_BUILD_BUILD_DIR=/root/.cargo-build",
+ "--mount",
+ f"type=bind,source={buildscripts_dir},target=/buildscripts,readonly",
+ "--mount",
+ f"type=bind,source={buildconfig_dir},target=/buildconfig,readonly",
+ "--mount",
+ f"type=bind,source={artifact_dir},target=/pkgdir",
+ "--mount",
+ f"type=bind,source={staging_dir},target=/out",
+ image,
+ "/buildscripts/sandcastle-build-generic",
+ component,
+ codename,
+ architecture,
+ ]
+ )
+ subprocess.run(command, check=True)
+
+ return run_build
+
+
+def parse_args(argv: Sequence[str]) -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--builder-image", default=BUILDER_IMAGE)
+ parser.add_argument("--distro", default=DISTRO)
+ parser.add_argument("--force", action="store_true")
+ return parser.parse_args(argv)
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+ args = parse_args(sys.argv[1:] if argv is None else argv)
+ root = Path(__file__).resolve().parents[1]
+ config_dir = root / "buildconfig"
+ generic_script = root / "buildscripts" / "sandcastle-build-generic"
+ artifact_dir = root / "packages" / args.distro
+ state_dir = root / "packages" / ".state" / args.distro
+ staging_root = root / "packages" / ".staging" / args.distro
+ cache_dir = root / "cache"
+
+ vendor, separator, codename = args.distro.partition("-")
+ if not separator or vendor != "debian" or not codename:
+ raise ValueError(f"unsupported distribution: {args.distro}")
+
+ builder_image_id, architecture = inspect_builder(args.builder_image)
+ artifact_dir /= architecture
+ state_dir /= architecture
+ staging_root /= architecture
+
+ cache_paths = (
+ cache_dir / "gradle",
+ cache_dir / "pnpm",
+ cache_dir / "cargo-registry",
+ cache_dir / "cargo-git",
+ cache_dir / "cargo-build",
+ cache_dir / args.distro / "apt-archives" / "partial",
+ cache_dir / args.distro / "apt-lists" / "partial",
+ artifact_dir,
+ state_dir,
+ staging_root,
+ )
+ ensure_directories(cache_paths)
+
+ order = topological_order(COMPONENTS, DEPENDENCIES)
+ component_inputs = {
+ component: component_input_digest(config_dir, component) for component in order
+ }
+ keys = desired_build_keys(
+ order,
+ DEPENDENCIES,
+ component_inputs,
+ distro=args.distro,
+ architecture=architecture,
+ builder_image_id=builder_image_id,
+ build_script_digest=file_digest(generic_script),
+ )
+ runner = podman_build_runner(
+ root=root,
+ image=args.builder_image,
+ architecture=architecture,
+ distro=args.distro,
+ codename=codename,
+ artifact_dir=artifact_dir,
+ cache_dir=cache_dir,
+ )
+ rebuilt = rebuild_outdated(
+ order,
+ keys,
+ artifact_dir,
+ state_dir,
+ staging_root,
+ runner,
+ force=args.force,
+ )
+
+ missing = [
+ component
+ for component in order
+ if not component_is_current(artifact_dir, state_dir, component, keys[component])
+ ]
+ if missing:
+ raise RuntimeError("packages are not current: " + ", ".join(missing))
+
+ if rebuilt:
+ print("Rebuilt packages: " + ", ".join(rebuilt))
+ else:
+ print("All packages are up to date")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/sandcastle-build b/sandcastle-build
@@ -2,30 +2,71 @@
set -eu
-no_cache=
+no_cache_args=()
+package_args=()
+containerfile=Dockerfile
+containerfile_set=
+
while [[ $# -gt 0 ]]; do
case $1 in
- --no-cache) no_cache=--no-cache ;;
- *)
- echo "Unknown option '$1'"
+ --no-cache)
+ no_cache_args=(--no-cache)
+ package_args=(--force)
+ ;;
+ -*)
+ echo >&2 "Unknown option '$1'"
exit 1
;;
+ *)
+ if [[ -n $containerfile_set ]]; then
+ echo >&2 "Unexpected argument '$1'"
+ exit 1
+ fi
+ containerfile=$1
+ containerfile_set=yes
+ ;;
esac
shift
done
-CONTAINERFILE="${1:-Dockerfile}"
-[ $# -gt 0 ] && shift
-
SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
cd "$SCRIPT_DIR"
-if [ ! -r "${CONTAINERFILE}" ]; then
- echo >&2 "Containerfile '${CONTAINERFILE}' not found or not readable."
+if [[ ! -r $containerfile ]]; then
+ echo >&2 "Containerfile '${containerfile}' not found or not readable."
exit 1
fi
-echo "Building with containerfile: ${CONTAINERFILE}"
+
+builder_image=localhost/taler-sandcastle-package-builder:latest
+cache_dir=$SCRIPT_DIR/cache
+mkdir -p \
+ "$cache_dir/debian-trixie/apt-archives/partial" \
+ "$cache_dir/debian-trixie/apt-lists/partial"
+
# The nofile ulimit is required to prevent fakeroot from becoming sluggish.
-exec podman build --log-level=debug $no_cache --ulimit=nofile=2048:2048 -f "${CONTAINERFILE}" \
- --target taler-final -t taler-base-all "$@" .
+build_args=(
+ podman build
+ --log-level=debug
+ --ulimit=nofile=2048:2048
+ -f "$containerfile"
+)
+
+echo "Building package-builder image with containerfile: $containerfile"
+"${build_args[@]}" "${no_cache_args[@]}" \
+ --volume "$cache_dir/debian-trixie/apt-archives:/var/cache/apt/archives:z" \
+ --volume "$cache_dir/debian-trixie/apt-lists:/var/lib/apt/lists:z" \
+ --target base-system \
+ --tag "$builder_image" \
+ .
+
+echo "Building outdated Debian packages"
+python3 buildscripts/sandcastle_build_packages.py \
+ --builder-image "$builder_image" \
+ "${package_args[@]}"
+
+echo "Building final sandcastle image"
+exec "${build_args[@]}" "${no_cache_args[@]}" \
+ --target taler-final \
+ --tag taler-base-all \
+ .
diff --git a/tests/test_sandcastle_build_packages.py b/tests/test_sandcastle_build_packages.py
@@ -0,0 +1,296 @@
+# This file is in the public domain.
+
+import os
+import runpy
+import shutil
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from buildscripts.sandcastle_build_packages import (
+ COMPONENTS,
+ DEPENDENCIES,
+ component_input_digest,
+ component_is_current,
+ desired_build_keys,
+ podman_build_runner,
+ rebuild_outdated,
+ topological_order,
+)
+
+
+class DebianVersionTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ script = (
+ Path(__file__).resolve().parents[1]
+ / "buildscripts"
+ / "sandcastle-build-generic"
+ )
+ cls.generic = runpy.run_path(str(script))
+
+ def test_development_tag_keeps_the_complete_sequence_number(self):
+ self.assertEqual("1.2.3~dev17", self.generic["get_tag_debver"]("v1.2.3-dev.17"))
+
+ def test_debian_revision_and_codename_are_preserved(self):
+ version = self.generic["get_tag_debver"]("deb-v1.2.3-4")
+ with_codename = self.generic["make_codename_version"](version, "trixie")
+ self.assertEqual("1.2.3-4+trixie", with_codename)
+
+
+class DependencyKeyTests(unittest.TestCase):
+ dependencies = {"b": ("a",), "c": ("b",)}
+ components = ("c", "d", "b", "a")
+
+ def keys(self, inputs):
+ order = topological_order(self.components, self.dependencies)
+ return desired_build_keys(
+ order,
+ self.dependencies,
+ inputs,
+ distro="debian-trixie",
+ architecture="amd64",
+ builder_image_id="builder-1",
+ build_script_digest="script-1",
+ )
+
+ def test_topological_order_puts_dependencies_first(self):
+ order = topological_order(self.components, self.dependencies)
+ self.assertLess(order.index("a"), order.index("b"))
+ self.assertLess(order.index("b"), order.index("c"))
+ self.assertEqual(set(self.components), set(order))
+
+ def test_changed_package_invalidates_only_it_and_reverse_dependencies(self):
+ before = self.keys({"a": "a1", "b": "b1", "c": "c1", "d": "d1"})
+ after = self.keys({"a": "a2", "b": "b1", "c": "c1", "d": "d1"})
+ changed = {
+ component for component in before if before[component] != after[component]
+ }
+ self.assertEqual({"a", "b", "c"}, changed)
+
+ def test_changed_consumer_does_not_invalidate_its_dependency(self):
+ before = self.keys({"a": "a1", "b": "b1", "c": "c1", "d": "d1"})
+ after = self.keys({"a": "a1", "b": "b2", "c": "c1", "d": "d1"})
+ changed = {
+ component for component in before if before[component] != after[component]
+ }
+ self.assertEqual({"b", "c"}, changed)
+
+ def test_cycles_are_rejected(self):
+ with self.assertRaisesRegex(ValueError, "cycle"):
+ topological_order(("a", "b"), {"a": ("b",), "b": ("a",)})
+
+ def test_exchange_change_rebuilds_its_real_reverse_dependencies(self):
+ order = topological_order(COMPONENTS, DEPENDENCIES)
+ inputs = {component: "unchanged" for component in COMPONENTS}
+ before = desired_build_keys(
+ order,
+ DEPENDENCIES,
+ inputs,
+ distro="debian-trixie",
+ architecture="amd64",
+ builder_image_id="builder-1",
+ build_script_digest="script-1",
+ )
+ inputs["taler-exchange"] = "changed"
+ after = desired_build_keys(
+ order,
+ DEPENDENCIES,
+ inputs,
+ distro="debian-trixie",
+ architecture="amd64",
+ builder_image_id="builder-1",
+ build_script_digest="script-1",
+ )
+ changed = {
+ component
+ for component in COMPONENTS
+ if before[component] != after[component]
+ }
+ self.assertEqual(
+ {
+ "taler-exchange",
+ "donau",
+ "taler-merchant",
+ "paivana-httpd",
+ "challenger",
+ },
+ changed,
+ )
+
+
+class ComponentInputDigestTests(unittest.TestCase):
+ def test_checkout_pin_changes_component_digest(self):
+ with tempfile.TemporaryDirectory() as temporary:
+ config_dir = Path(temporary)
+ (config_dir / "component.tag").write_text("master\n", encoding="utf-8")
+ (config_dir / "component.giturl").write_text(
+ "https://example.com/component.git\n", encoding="utf-8"
+ )
+
+ without_checkout = component_input_digest(config_dir, "component")
+ (config_dir / "component.checkout").write_text(
+ "first-commit\n", encoding="utf-8"
+ )
+ first_checkout = component_input_digest(config_dir, "component")
+ (config_dir / "component.checkout").write_text(
+ "second-commit\n", encoding="utf-8"
+ )
+ second_checkout = component_input_digest(config_dir, "component")
+
+ self.assertNotEqual(without_checkout, first_checkout)
+ self.assertNotEqual(first_checkout, second_checkout)
+
+
+class ArtifactStateTests(unittest.TestCase):
+ def setUp(self):
+ self.temporary = tempfile.TemporaryDirectory()
+ root = Path(self.temporary.name)
+ self.artifacts = root / "artifacts"
+ self.state = root / "state"
+ self.staging = root / "staging"
+ self.artifacts.mkdir()
+ self.state.mkdir()
+ self.staging.mkdir()
+
+ def tearDown(self):
+ self.temporary.cleanup()
+
+ def record_old_build(self):
+ component_dir = self.artifacts / "a"
+ component_dir.mkdir()
+ (component_dir / "old.deb").write_bytes(b"old")
+ (self.state / "a.key").write_text("old-key\n", encoding="utf-8")
+
+ def test_failed_build_keeps_previous_artifacts_and_key(self):
+ self.record_old_build()
+
+ def fail_build(_component, staging_dir):
+ (staging_dir / "new.deb").write_bytes(b"new")
+ raise RuntimeError("build failed")
+
+ with self.assertRaisesRegex(RuntimeError, "build failed"):
+ rebuild_outdated(
+ ["a"],
+ {"a": "new-key"},
+ self.artifacts,
+ self.state,
+ self.staging,
+ fail_build,
+ )
+
+ self.assertEqual(b"old", (self.artifacts / "a" / "old.deb").read_bytes())
+ self.assertEqual("old-key", (self.state / "a.key").read_text().strip())
+ self.assertFalse(
+ component_is_current(self.artifacts, self.state, "a", "new-key")
+ )
+ self.assertEqual([], list(self.staging.iterdir()))
+
+ def test_successful_build_promotes_artifacts_and_key(self):
+ self.record_old_build()
+
+ def succeed_build(_component, staging_dir):
+ (staging_dir / "new.deb").write_bytes(b"new")
+
+ rebuilt = rebuild_outdated(
+ ["a"],
+ {"a": "new-key"},
+ self.artifacts,
+ self.state,
+ self.staging,
+ succeed_build,
+ )
+
+ self.assertEqual(["a"], rebuilt)
+ self.assertFalse((self.artifacts / "a" / "old.deb").exists())
+ self.assertEqual(b"new", (self.artifacts / "a" / "new.deb").read_bytes())
+ self.assertTrue(
+ component_is_current(self.artifacts, self.state, "a", "new-key")
+ )
+
+ def test_matching_key_without_an_artifact_is_not_current(self):
+ (self.state / "a.key").write_text("key\n", encoding="utf-8")
+ self.assertFalse(component_is_current(self.artifacts, self.state, "a", "key"))
+
+
+class PackageRunnerTests(unittest.TestCase):
+ @patch("buildscripts.sandcastle_build_packages.subprocess.run")
+ def test_package_container_has_bounded_nofile_limit(self, run):
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ runner = podman_build_runner(
+ root=root,
+ image="builder-image",
+ architecture="amd64",
+ distro="debian-trixie",
+ codename="trixie",
+ artifact_dir=root / "artifacts",
+ cache_dir=root / "cache",
+ )
+
+ runner("component", root / "staging")
+
+ command = run.call_args.args[0]
+ self.assertIn("--ulimit=nofile=2048:2048", command)
+ run.assert_called_once_with(command, check=True)
+
+
+class BuildOrchestrationTests(unittest.TestCase):
+ def test_build_phases_and_no_cache_propagation(self):
+ repository = Path(__file__).resolve().parents[1]
+ with tempfile.TemporaryDirectory() as temporary:
+ root = Path(temporary)
+ (root / "bin").mkdir()
+ (root / "buildscripts").mkdir()
+ shutil.copy(repository / "sandcastle-build", root / "sandcastle-build")
+ (root / "Dockerfile").write_text("FROM scratch\n", encoding="utf-8")
+ (root / "buildscripts" / "sandcastle_build_packages.py").touch()
+ log = root / "calls"
+
+ fake_command = '#!/bin/sh\nprintf \'%s\' "$0" >> "$CALL_LOG"\nprintf \' <%s>\' "$@" >> "$CALL_LOG"\nprintf \'\\n\' >> "$CALL_LOG"\n'
+ for command in ("podman", "python3"):
+ executable = root / "bin" / command
+ executable.write_text(fake_command, encoding="utf-8")
+ executable.chmod(0o755)
+
+ environment = os.environ.copy()
+ environment["PATH"] = f"{root / 'bin'}:{environment['PATH']}"
+ environment["CALL_LOG"] = str(log)
+ subprocess.run(
+ [str(root / "sandcastle-build"), "--no-cache"],
+ check=True,
+ cwd=root,
+ env=environment,
+ stdout=subprocess.PIPE,
+ text=True,
+ )
+
+ calls = log.read_text(encoding="utf-8").splitlines()
+ self.assertEqual(3, len(calls))
+ self.assertIn("<--target> <base-system>", calls[0])
+ self.assertIn("<--no-cache>", calls[0])
+ self.assertIn("<buildscripts/sandcastle_build_packages.py>", calls[1])
+ self.assertIn("<--force>", calls[1])
+ self.assertIn("<--target> <taler-final>", calls[2])
+ self.assertIn("<--no-cache>", calls[2])
+
+ log.write_text("", encoding="utf-8")
+ subprocess.run(
+ [str(root / "sandcastle-build")],
+ check=True,
+ cwd=root,
+ env=environment,
+ stdout=subprocess.PIPE,
+ text=True,
+ )
+ calls = log.read_text(encoding="utf-8").splitlines()
+ self.assertEqual(3, len(calls))
+ self.assertNotIn("<--no-cache>", calls[0])
+ self.assertNotIn("<--force>", calls[1])
+ self.assertNotIn("<--no-cache>", calls[2])
+
+
+if __name__ == "__main__":
+ unittest.main()
