commit c8a1d9fc3aa8aa590cd1aa547d7a43c828b87333 parent 15b8639571a77ecdab2237207d66f6577bc07c76 Author: Florian Dold <dold@taler.net> Date: Tue, 25 Aug 2026 00:59:09 +0200 packaging: group pnpm workspace Debian builds Diffstat:
70 files changed, 1319 insertions(+), 522 deletions(-)
diff --git a/packaging/ng/README.md b/packaging/ng/README.md @@ -50,10 +50,10 @@ apt-get install qemu-user-static binfmt-support ## Structure * `packages/$DISTRO-$DISTRO_VERNAME`: Output folder for debian packages. -Also contains a `Packages.gz` metadata file generated by `dpkg-scanpackages` +Also contains a `Packages.xz` metadata file generated by `dpkg-scanpackages` so that this folder can be directly consumed as a trusted package source. -* `buildconfig/*`: Per-component configuration, see "Versions" below. +* `packages.toml`: Repository, package, version, builder, and dependency configuration. * `buildscripts/*`: Build scripts used during the package build steps. * `distros/*`: Files for building for a specific distro. * `server-side/*.sh`: Scripts that must be installed in taler-packaing@taler.net:$HOME @@ -61,21 +61,40 @@ so that this folder can be directly consumed as a trusted package source. ## Versions -Each component is configured by these files in `buildconfig/`: - -* `$component.giturl`: the upstream repository. -* `$component.tag`: the git tag that is built. It determines the version of the -resulting debian package. -* `$component.debpath`: the directory within the repository that contains the -`debian/` folder, relative to the repository root. Optional, defaults to the -repository root itself. - -`debpath` exists so that one repository can provide several packages that are -built independently. Their tag files are separate, but the tags all come from -the same repository, so `upgrade` moves all of them to the same tag. This -affects every component built from `taler-typescript-core` -(`taler-wallet-cli`, `taler-harness` and the `*-webui` packages), each of which -has a `packages/$component` debpath. +`packages.toml` is the single source of truth. A repository entry defines its +URL and default builder. A package entry selects a repository and tag, and may +also define `debian_path`, `dependencies`, `enabled`, or override `builder`. +Missing paths and dependency lists default to empty, packages default to +enabled, and builders default to `generic`. + +```toml +[repositories."typescript-core"] +url = "git://git.taler.net/taler-typescript-core.git" +builder = "pnpm-workspace" + +[packages."taler-wallet-cli"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-wallet-cli" +dependencies = [] +enabled = true +``` + +The `pnpm-workspace` builder groups packages only when their effective +repository URL, tag, and builder are identical. It clones and bootstraps once, +installs and builds the union of the selected workspace dependency closures, +and then produces each Debian package separately. Packages from one repository +may use different tags; they are put into different build groups. + +During the individual Debian builds, this builder sets +`TALER_PACKAGING_PREBUILT=1`. Workspace package Makefiles use it to skip their +normal dependency/build step and install the artifacts prepared by the grouped +build instead. Standalone `make install` behavior is unchanged. + +Configuration is parsed with Python's standard `tomllib`. `taler-pkg upgrade` +rewrites it in canonical order using a schema-specific serializer and validates +the result before atomically replacing the file. Manual formatting and custom +comments are therefore not preserved across upgrades. Supported tag syntax (see `buildscripts/generic`): @@ -97,7 +116,7 @@ v1.2.3-dev.1 < v1.2.3-dev.2 < v1.2.3 < deb-v1.2.3-1 < v1.2.4-dev.1 ### From git tag to debian version -`buildscripts/generic` derives the debian version of a package from the tag in +The shared build library derives the debian version of a package from the tag in two steps. `get_tag_debver` strips the `v` / `deb-v` prefix and rewrites `-dev.$n` into `~dev$n`, which sorts before the corresponding release. `make_codename_version` then appends the distro codename. If the version has no @@ -116,7 +135,7 @@ deb-v1.2.3-1 => 1.2.3-1+trixie ``` The resulting versions sort the same way as the tags they come from, so bumping -a tag file always produces a package that apt considers newer: +a package tag always produces a package that apt considers newer: ``` 1.2.3~dev1-0+trixie < 1.2.3~dev10-0+trixie < 1.2.3-0+trixie < 1.2.3-1+trixie @@ -137,7 +156,7 @@ a tag file always produces a package that apt considers newer: # Show current vs. latest version ./taler-pkg show-latest -# Show which tag files would change +# Show which package tags would change ./taler-pkg upgrade --dry # Bump all components to the latest release tag @@ -150,27 +169,30 @@ a tag file always produces a package that apt considers newer: ./taler-pkg upgrade --dev # Set a version manually -echo $desired_version > buildconfig/$component.tag +# Edit the package's tag in packages.toml ``` -`upgrade` looks at every `buildconfig/*.tag` file, queries the corresponding -repository for its tags and writes back the newest one. Without `--dev`, only -release and `deb-v` tags are considered, with `--dev` also dev tags. +`upgrade` considers every package in `packages.toml`, including disabled ones, +queries each distinct repository URL once, and writes back the newest tag for +each selected package. Without `--dev`, only release and `deb-v` tags are +considered, with `--dev` also dev tags. -Components are only bumped forwards. A tag file that already points to a -version newer than the newest candidate is left alone, which is what happens to -components pinned to a dev tag when running without `--dev`. Tag files with -unsupported syntax and components without a `.giturl` are also left alone. +Components are only bumped forwards. A package already pointing to a version +newer than the newest candidate is left alone, which is what happens to packages +pinned to a dev tag when running without `--dev`. Unsupported tag syntax is +also left alone. -Components that share a repository, i.e. those distinguished by a `debpath`, -always end up on the same tag. +Packages sharing a repository are updated independently. Passing package names +to `upgrade` can intentionally leave packages from one repository on different +tags. -`upgrade` only changes tag files; run `./taler-pkg build $DISTRO` afterwards. +`upgrade` only changes `packages.toml`; run `./taler-pkg build $DISTRO` +afterwards. ### Forcing a rebuild ``` -rm packages/$distro/$component.built.tag +rm packages/$distro/$component@$arch.built.tag ``` ### Adding a new distro diff --git a/packaging/ng/buildconfig/anastasis-gtk.giturl b/packaging/ng/buildconfig/anastasis-gtk.giturl @@ -1 +0,0 @@ -git://git.taler.net/anastasis-gtk.git diff --git a/packaging/ng/buildconfig/anastasis-gtk.tag b/packaging/ng/buildconfig/anastasis-gtk.tag @@ -1 +0,0 @@ -v0.8.1 diff --git a/packaging/ng/buildconfig/anastasis.giturl b/packaging/ng/buildconfig/anastasis.giturl @@ -1 +0,0 @@ -git://git.taler.net/anastasis.git diff --git a/packaging/ng/buildconfig/anastasis.tag b/packaging/ng/buildconfig/anastasis.tag @@ -1 +0,0 @@ -v0.8.1 diff --git a/packaging/ng/buildconfig/challenger-webui.debpath b/packaging/ng/buildconfig/challenger-webui.debpath @@ -1 +0,0 @@ -packages/challenger-webui diff --git a/packaging/ng/buildconfig/challenger-webui.giturl b/packaging/ng/buildconfig/challenger-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/challenger-webui.tag b/packaging/ng/buildconfig/challenger-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/challenger.giturl b/packaging/ng/buildconfig/challenger.giturl @@ -1 +0,0 @@ -git://git.taler.net/challenger.git diff --git a/packaging/ng/buildconfig/challenger.tag b/packaging/ng/buildconfig/challenger.tag @@ -1 +0,0 @@ -v1.6.3-dev.1 diff --git a/packaging/ng/buildconfig/depolymerization.giturl b/packaging/ng/buildconfig/depolymerization.giturl @@ -1 +0,0 @@ -git://git.taler.net/depolymerization.git diff --git a/packaging/ng/buildconfig/depolymerization.tag b/packaging/ng/buildconfig/depolymerization.tag @@ -1 +0,0 @@ -v0.0.1 diff --git a/packaging/ng/buildconfig/donau.giturl b/packaging/ng/buildconfig/donau.giturl @@ -1 +0,0 @@ -git://git.taler.net/donau.git diff --git a/packaging/ng/buildconfig/donau.tag b/packaging/ng/buildconfig/donau.tag @@ -1 +0,0 @@ -v1.6.7 diff --git a/packaging/ng/buildconfig/gnunet.giturl b/packaging/ng/buildconfig/gnunet.giturl @@ -1 +0,0 @@ -git://git.gnunet.org/gnunet.git diff --git a/packaging/ng/buildconfig/gnunet.tag b/packaging/ng/buildconfig/gnunet.tag @@ -1 +0,0 @@ -v0.28.1-dev.5 diff --git a/packaging/ng/buildconfig/libeufin-bank-webui.debpath b/packaging/ng/buildconfig/libeufin-bank-webui.debpath @@ -1 +0,0 @@ -packages/libeufin-bank-webui diff --git a/packaging/ng/buildconfig/libeufin-bank-webui.giturl b/packaging/ng/buildconfig/libeufin-bank-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/libeufin-bank-webui.tag b/packaging/ng/buildconfig/libeufin-bank-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/libeufin.giturl b/packaging/ng/buildconfig/libeufin.giturl @@ -1 +0,0 @@ -git://git.taler.net/libeufin.git diff --git a/packaging/ng/buildconfig/libeufin.tag b/packaging/ng/buildconfig/libeufin.tag @@ -1 +0,0 @@ -v1.6.9-dev.1 diff --git a/packaging/ng/buildconfig/paivana-httpd.giturl b/packaging/ng/buildconfig/paivana-httpd.giturl @@ -1 +0,0 @@ -git://git.taler.net/paivana.git diff --git a/packaging/ng/buildconfig/paivana-httpd.tag b/packaging/ng/buildconfig/paivana-httpd.tag @@ -1 +0,0 @@ -v1.6.3 diff --git a/packaging/ng/buildconfig/robocop.giturl b/packaging/ng/buildconfig/robocop.giturl @@ -1 +0,0 @@ -git://git.taler.net/robocop.git diff --git a/packaging/ng/buildconfig/robocop.tag b/packaging/ng/buildconfig/robocop.tag @@ -1 +0,0 @@ -v1.6.0 diff --git a/packaging/ng/buildconfig/sync.giturl b/packaging/ng/buildconfig/sync.giturl @@ -1 +0,0 @@ -git://git.taler.net/sync.git diff --git a/packaging/ng/buildconfig/sync.tag b/packaging/ng/buildconfig/sync.tag @@ -1 +0,0 @@ -v1.6.0-dev.1 diff --git a/packaging/ng/buildconfig/taler-auditor-webui.debpath b/packaging/ng/buildconfig/taler-auditor-webui.debpath @@ -1 +0,0 @@ -packages/taler-auditor-webui diff --git a/packaging/ng/buildconfig/taler-auditor-webui.giturl b/packaging/ng/buildconfig/taler-auditor-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/taler-auditor-webui.tag b/packaging/ng/buildconfig/taler-auditor-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/taler-challenger-helpers.giturl b/packaging/ng/buildconfig/taler-challenger-helpers.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-challenger-helpers.git diff --git a/packaging/ng/buildconfig/taler-challenger-helpers.tag b/packaging/ng/buildconfig/taler-challenger-helpers.tag @@ -1 +0,0 @@ -v1.6.2 diff --git a/packaging/ng/buildconfig/taler-directory.giturl b/packaging/ng/buildconfig/taler-directory.giturl @@ -1 +0,0 @@ -git://git.taler.net/taldir.git diff --git a/packaging/ng/buildconfig/taler-directory.tag b/packaging/ng/buildconfig/taler-directory.tag @@ -1 +0,0 @@ -v1.5.1 diff --git a/packaging/ng/buildconfig/taler-exchange-aml-webui.debpath b/packaging/ng/buildconfig/taler-exchange-aml-webui.debpath @@ -1 +0,0 @@ -packages/taler-exchange-aml-webui diff --git a/packaging/ng/buildconfig/taler-exchange-aml-webui.giturl b/packaging/ng/buildconfig/taler-exchange-aml-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/taler-exchange-aml-webui.tag b/packaging/ng/buildconfig/taler-exchange-aml-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/taler-exchange-kyc-webui.debpath b/packaging/ng/buildconfig/taler-exchange-kyc-webui.debpath @@ -1 +0,0 @@ -packages/taler-exchange-kyc-webui diff --git a/packaging/ng/buildconfig/taler-exchange-kyc-webui.giturl b/packaging/ng/buildconfig/taler-exchange-kyc-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/taler-exchange-kyc-webui.tag b/packaging/ng/buildconfig/taler-exchange-kyc-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/taler-exchange.giturl b/packaging/ng/buildconfig/taler-exchange.giturl @@ -1 +0,0 @@ -git://git.taler.net/exchange.git diff --git a/packaging/ng/buildconfig/taler-exchange.tag b/packaging/ng/buildconfig/taler-exchange.tag @@ -1 +0,0 @@ -v1.6.14 diff --git a/packaging/ng/buildconfig/taler-harness.debpath b/packaging/ng/buildconfig/taler-harness.debpath @@ -1 +0,0 @@ -packages/taler-harness diff --git a/packaging/ng/buildconfig/taler-harness.giturl b/packaging/ng/buildconfig/taler-harness.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/taler-harness.tag b/packaging/ng/buildconfig/taler-harness.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/taler-mailbox.giturl b/packaging/ng/buildconfig/taler-mailbox.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-mailbox.git diff --git a/packaging/ng/buildconfig/taler-mailbox.tag b/packaging/ng/buildconfig/taler-mailbox.tag @@ -1 +0,0 @@ -v1.5.1 diff --git a/packaging/ng/buildconfig/taler-mdb.giturl b/packaging/ng/buildconfig/taler-mdb.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-mdb.git diff --git a/packaging/ng/buildconfig/taler-mdb.tag b/packaging/ng/buildconfig/taler-mdb.tag @@ -1 +0,0 @@ -v1.6.0 diff --git a/packaging/ng/buildconfig/taler-merchant-demos.giturl b/packaging/ng/buildconfig/taler-merchant-demos.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-merchant-demos diff --git a/packaging/ng/buildconfig/taler-merchant-demos.tag b/packaging/ng/buildconfig/taler-merchant-demos.tag @@ -1 +0,0 @@ -v1.6.1 diff --git a/packaging/ng/buildconfig/taler-merchant-webui.debpath b/packaging/ng/buildconfig/taler-merchant-webui.debpath @@ -1 +0,0 @@ -packages/taler-merchant-webui diff --git a/packaging/ng/buildconfig/taler-merchant-webui.giturl b/packaging/ng/buildconfig/taler-merchant-webui.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/taler-merchant-webui.tag b/packaging/ng/buildconfig/taler-merchant-webui.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildconfig/taler-merchant.giturl b/packaging/ng/buildconfig/taler-merchant.giturl @@ -1 +0,0 @@ -git://git.taler.net/merchant.git diff --git a/packaging/ng/buildconfig/taler-merchant.tag b/packaging/ng/buildconfig/taler-merchant.tag @@ -1 +0,0 @@ -v1.6.19 diff --git a/packaging/ng/buildconfig/taler-rust.giturl b/packaging/ng/buildconfig/taler-rust.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-rust.git diff --git a/packaging/ng/buildconfig/taler-rust.tag b/packaging/ng/buildconfig/taler-rust.tag @@ -1 +0,0 @@ -v1.5.0 diff --git a/packaging/ng/buildconfig/taler-wallet-cli.debpath b/packaging/ng/buildconfig/taler-wallet-cli.debpath @@ -1 +0,0 @@ -packages/taler-wallet-cli diff --git a/packaging/ng/buildconfig/taler-wallet-cli.giturl b/packaging/ng/buildconfig/taler-wallet-cli.giturl @@ -1 +0,0 @@ -git://git.taler.net/taler-typescript-core.git diff --git a/packaging/ng/buildconfig/taler-wallet-cli.tag b/packaging/ng/buildconfig/taler-wallet-cli.tag @@ -1 +0,0 @@ -v1.6.33-dev.2 diff --git a/packaging/ng/buildscripts/buildlib.py b/packaging/ng/buildscripts/buildlib.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 + +# This file is in the public domain. + +import json +import os +import shutil +import subprocess +import sys +from email.utils import formatdate +from pathlib import Path + +from package_config import ConfigError, load_config + +PKGDIR = Path("/pkgdir") + + +def run_cmd(cmd, shell=False, cwd=None, env=None): + """Run a command and stop the build if it fails.""" + command_env = os.environ.copy() + if env: + command_env.update(env) + sys.stdout.flush() + subprocess.check_call(cmd, shell=shell, cwd=cwd, env=command_env) + + +def get_output(cmd, shell=False, cwd=None): + return subprocess.check_output(cmd, shell=shell, cwd=cwd, text=True).strip() + + +def remove_stale_packages(pkgdir): + """Remove package artifacts not referenced by a current-build manifest.""" + pkgdir = Path(pkgdir) + current_packages = set() + for manifest in pkgdir.glob("*.built.current"): + current_packages.update(manifest.read_text().split()) + + for artifact in sorted(pkgdir.iterdir()): + if artifact.suffix not in (".deb", ".ddeb"): + continue + if artifact.name in current_packages: + continue + print(f"Removing stale local package {artifact.name}") + artifact.unlink() + + +def get_tag_debver(tag): + """Get a Debian version string from a supported Git tag.""" + if tag.startswith("v"): + devsuff = "-dev." + position = tag.find(devsuff) + if position < 0: + return tag[1:] + return tag[1:position] + "~dev" + tag[position + len(devsuff) :] + if tag.startswith("deb-v"): + tag = tag[5:] + if "-" in tag: + version, revision = tag.split("-", 1) + return version + "-" + revision + return tag + raise ValueError(f"unexpected tag format: {tag}") + + +def make_codename_version(deb_version, build_codename): + if "-" in deb_version: + return f"{deb_version}+{build_codename}" + return f"{deb_version}-0+{build_codename}" + + +def scan_local_repository(): + os.chdir(PKGDIR) + packages_index = PKGDIR / "Packages.xz" + run_cmd(f"dpkg-scanpackages . | xz - > {packages_index}", shell=True) + with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as source: + source.write(f"deb [trusted=yes] file:{PKGDIR} ./\n") + run_cmd(["apt-get", "update"]) + + +def _validate_group(config, package_names, expected_builder): + if not package_names: + raise ConfigError("no packages specified") + if len(set(package_names)) != len(package_names): + raise ConfigError("a package was specified more than once") + + packages = [] + keys = set() + for name in package_names: + try: + package = config.packages[name] + except KeyError as exc: + raise ConfigError(f"unknown package {name!r}") from exc + repository = config.repository_for(package) + builder = config.builder_for(package) + keys.add((repository.url, package.tag, builder)) + packages.append(package) + if len(keys) != 1: + raise ConfigError("grouped packages must have the same repository, tag, and builder") + repository_url, tag, builder = keys.pop() + if builder != expected_builder: + raise ConfigError(f"expected builder {expected_builder!r}, got {builder!r}") + if expected_builder == "generic" and len(packages) != 1: + raise ConfigError("the generic builder accepts exactly one package") + return packages, repository_url, tag + + +def _install_build_dependencies(source_dir, package_paths): + tool = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" + controls = [str(package_path / "debian" / "control") for package_path in package_paths] + run_cmd( + ["mk-build-deps", "--install", f"--tool={tool}", *controls], + cwd=source_dir, + ) + + +def _prepare_pnpm_workspace(source_dir, package_paths): + filters = [] + for package_path in package_paths: + package_json = package_path / "package.json" + with package_json.open(encoding="utf-8") as package_file: + workspace_name = json.load(package_file).get("name") + if not isinstance(workspace_name, str) or not workspace_name: + raise ConfigError(f"{package_json} has no package name") + filters.extend(["--filter", f"{workspace_name}..."]) + + run_cmd(["pnpm", "install", "--frozen-lockfile", *filters], cwd=source_dir) + run_cmd(["pnpm", "run", *filters, "build"], cwd=source_dir) + + +def _write_changelog(package_path, package_name, version): + debian_date = formatdate(localtime=True) + changelog = f"""\ +{package_name} ({version}) unstable; urgency=low + + * Release {version}. + + -- Taler Packaging Team <deb@taler.net> {debian_date} +""" + (package_path / "debian" / "changelog").write_text(changelog) + + +def _package_artifacts(output_dir): + return { + artifact.resolve() + for pattern in ("*.deb", "*.ddeb") + for artifact in output_dir.glob(pattern) + } + + +def _check_installed_binaries(deb_files): + for deb in deb_files: + contents = get_output(["dpkg", "--contents", str(deb)]) + for line in contents.splitlines(): + parts = line.split() + if len(parts) < 6: + raise RuntimeError(f"failed to read package contents from {deb}") + filename = parts[5] + if "bin" not in filename: + continue + if filename.startswith("./"): + filename = filename[2:] + if not filename.startswith("/"): + filename = "/" + filename + file_info = get_output(["file", filename]) + if "ELF" in file_info and "executable" in file_info: + print(f"checking {filename}") + try: + run_cmd(["ldd", filename]) + except subprocess.CalledProcessError as exc: + raise RuntimeError( + f"installed binary {filename} has a linker issue" + ) from exc + + +def _build_package(package, package_path, tag, codename, arch, prebuilt): + version = make_codename_version(get_tag_debver(tag), codename) + print(f"Building {package.name} as version {version}", file=sys.stderr) + _write_changelog(package_path, package.name, version) + + output_dir = package_path.parent + before = _package_artifacts(output_dir) + environment = {"DEB_BUILD_MAINT_OPTIONS": "debug"} + if prebuilt: + environment["TALER_PACKAGING_PREBUILT"] = "1" + + debug_repository = package_path / "debian" / ".debhelper" + debug_repository.mkdir(parents=True, exist_ok=True) + (debug_repository / "debian-symbols-pool").touch() + environment["DEB_DBG_SYMBOLS_REPO"] = "debian/.debhelper/" + + run_cmd( + ["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"], + cwd=package_path, + env=environment, + ) + artifacts = _package_artifacts(output_dir) - before + deb_files = sorted(path for path in artifacts if path.suffix == ".deb") + if not deb_files: + raise RuntimeError(f"{package.name} did not produce a Debian package") + + print(f"Installing built packages from {output_dir}", file=sys.stderr) + run_cmd(["apt", "install", "-y", *map(str, deb_files)]) + _check_installed_binaries(deb_files) + + for artifact in artifacts: + shutil.copy(artifact, PKGDIR) + manifest = PKGDIR / f"{package.name}@{arch}.built.current" + manifest.write_text("".join(f"{artifact.name}\n" for artifact in sorted(artifacts))) + (PKGDIR / f"{package.name}@{arch}.built.tag").write_text(tag + "\n") + remove_stale_packages(PKGDIR) + scan_local_repository() + + +def build_packages(codename, arch, package_names, expected_builder): + if "LD_LIBRARY_PATH" in os.environ: + del os.environ["LD_LIBRARY_PATH"] + + config = load_config("/packages.toml") + packages, repository_url, tag = _validate_group( + config, package_names, expected_builder + ) + print( + f"Building {' '.join(package_names)} with {expected_builder} build logic", + file=sys.stderr, + ) + + scan_local_repository() + source_dir = Path("/build/source") + source_dir.parent.mkdir(parents=True, exist_ok=True) + run_cmd(["git", "config", "--global", "advice.detachedHead", "false"]) + run_cmd( + [ + "git", + "clone", + "--depth=1", + f"--branch={tag}", + repository_url, + str(source_dir), + ] + ) + run_cmd(["./bootstrap"], cwd=source_dir) + + package_paths = [source_dir / package.debian_path for package in packages] + for package, package_path in zip(packages, package_paths): + control = package_path / "debian" / "control" + if not control.is_file(): + raise ConfigError( + f"{package.name} has no debian/control at tag {tag}: {control}" + ) + (package_path / ".version").write_text(get_tag_debver(tag) + "\n") + + _install_build_dependencies(source_dir, package_paths) + if expected_builder == "pnpm-workspace": + _prepare_pnpm_workspace(source_dir, package_paths) + + for package, package_path in zip(packages, package_paths): + _build_package( + package, + package_path, + tag, + codename, + arch, + prebuilt=expected_builder == "pnpm-workspace", + ) + + +def main(expected_builder): + if len(sys.argv) < 4: + print( + f"Usage: {Path(sys.argv[0]).name} <CODENAME> <ARCH> <PACKAGE>...", + file=sys.stderr, + ) + sys.exit(1) + try: + build_packages(sys.argv[1], sys.argv[2], sys.argv[3:], expected_builder) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/packaging/ng/buildscripts/generic b/packaging/ng/buildscripts/generic @@ -1,276 +1,14 @@ #!/usr/bin/env python3 -# This file is in the public domain. -# Helper script to build the latest DEB packages in the container. -# Supported tag syntax variants: -# v$maj.$min$.$patch => release version -# v$maj.$min$.$patch-dev.$n => dev version -# deb-v$maj.$min$.$patch-$revision => release version with debian revision -# Debian revisions of dev versions are *not* supported +# This file is in the public domain. -import os import sys -import subprocess -import glob -import shutil -from email.utils import formatdate from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) -def run_cmd(cmd, shell=False, cwd=None, env=None): - """Helper to run commands and exit on failure (mimicking set -e).""" - # If specific env vars are passed, update the current env, otherwise use current - command_env = os.environ.copy() - if env: - command_env.update(env) - - # Flush stdout so logs appear in order - sys.stdout.flush() - - subprocess.check_call(cmd, shell=shell, cwd=cwd, env=command_env) - - -def get_output(cmd, shell=False, cwd=None): - """Helper to get command output as string.""" - return subprocess.check_output(cmd, shell=shell, cwd=cwd, text=True).strip() - - -def remove_stale_packages(pkgdir): - """Remove package artifacts not referenced by a current-build manifest.""" - pkgdir = Path(pkgdir) - current_packages = set() - for manifest in pkgdir.glob("*.built.current"): - current_packages.update(manifest.read_text().split()) - - for artifact in sorted(pkgdir.iterdir()): - if artifact.suffix not in (".deb", ".ddeb"): - continue - if artifact.name in current_packages: - continue - print(f"Removing stale local package {artifact.name}") - artifact.unlink() - - -def get_tag_debver(tag): - """Get a debian version string from a git tag""" - if tag.startswith("v"): - devsuff = "-dev." - d = tag.find(devsuff) - if d < 0: - return tag[1:] - else: - return tag[1:d] + "~dev" + tag[d + len(devsuff):] - if tag.startswith("deb-v"): - tag = tag[5:] - if "-" in tag: - a, b = tag.split("-") - return a + "-" + b - return tag - raise ValueError(f"unexpected tag format: {tag}") - - - -def make_codename_version(deb_version, build_codename): - if "-" in deb_version: - # We already have a dash, thus a debian version. - # Append out build_codename to that. - return f"{deb_version}+{build_codename}" - # We don't have a debian version yet, start at zero, because by convention, - # real / manually created debian release versions start at 1. - return f"{deb_version}-0+{build_codename}" - - -def main(): - if 'LD_LIBRARY_PATH' in os.environ: - del os.environ['LD_LIBRARY_PATH'] - - # Arguments - if len(sys.argv) < 3: - print("Usage: script.py <PACKAGE> <CODENAME> <ARCH>", file=sys.stderr) - sys.exit(1) - - PACKAGE = sys.argv[1] - CODENAME = sys.argv[2] - ARCH = sys.argv[3] - - # Path of the debian/ folder in the repository - DEBIANPATH = "" - debpath_file = f"/buildconfig/{PACKAGE}.debpath" - if os.path.exists(debpath_file): - with open(debpath_file, 'r') as f: - DEBIANPATH = f.read().strip() - - print(f"Building {PACKAGE} with generic build logic", file=sys.stderr) - - with open(f"/buildconfig/{PACKAGE}.tag", 'r') as f: - TAG = f.read().strip() - - # 1. Setup Local Repo - os.chdir("/pkgdir") - # Using shell=True here to easily handle the pipe logic - run_cmd("dpkg-scanpackages . | xz - > /pkgdir/Packages.xz", shell=True) - - with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as f: - f.write("deb [trusted=yes] file:/pkgdir ./\n") - - run_cmd(["apt-get", "update"]) - - # 2. Prepare Build Directory - if not os.path.exists("/build"): - os.makedirs("/build") - os.chdir("/build") - - with open(f"/buildconfig/{PACKAGE}.giturl", 'r') as f: - GITURL = f.read().strip() - - run_cmd(["git", "config", "--global", "advice.detachedHead", "false"]) - run_cmd(["git", "clone", "--depth=1", f"--branch={TAG}", GITURL, PACKAGE]) - - build_pkg_path = os.path.join("/build", PACKAGE, DEBIANPATH) - os.chdir(build_pkg_path) - - DEB_VERSION = get_tag_debver(TAG) - - # 3. Bootstrap and Install Deps - os.chdir(os.path.join("/build", PACKAGE)) - run_cmd(["./bootstrap"]) - - os.chdir(build_pkg_path) - - # Install build-time dependencies - # Note: Passing the tool command as a single string because it contains spaces/options - tool_cmd = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes" - run_cmd(["mk-build-deps", "--install", f"--tool={tool_cmd}", "debian/control"]) - - # Sparse checkout hint - with open(".version", "w") as f: - f.write(f"{DEB_VERSION}\n") - - # 4. Configure Environment for Build - deb_dbg_repo = "debian/.debhelper/" - os.environ["DEB_DBG_SYMBOLS_REPO"] = deb_dbg_repo - os.makedirs(deb_dbg_repo, exist_ok=True) - - # touch command - open(os.path.join(deb_dbg_repo, "debian-symbols-pool"), 'a').close() - - os.environ["DEB_BUILD_MAINT_OPTIONS"] = "debug" - - # Add a codename-specific tag to the package. - # We need this tag as reprepro doesn't support multiple packages (even for - # different codenames like trixie/bookworm) per repo. - # Examples of how it should work: - # 1.2.3 => 1.2.3-0~trixie - # 1.2.3-2 => 1.2.3-2~trixie - # What we must not do: - # WRONG: 1.2.3 => 1.2.3~trixie - # Because 1.2.3~trixie <= 1.2.3, version requirements in dependencies would - # not work. - - out_deb_version = make_codename_version(DEB_VERSION, CODENAME) - - print(f"Building {PACKAGE} as version {out_deb_version}", file=sys.stderr) - - debian_date = formatdate(localtime=True) - - changelog = f"""\ -{PACKAGE} ({out_deb_version}) unstable; urgency=low - - * Release {out_deb_version}. - - -- Taler Packaging Team <deb@taler.net> {debian_date} -""" - - with open("debian/changelog", "w") as f: - f.write(changelog) - - # 5. Build Package - run_cmd(["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"]) - - # Copy artifacts - # Globs for ../*.deb and ../*.ddeb relative to current dir - deb_files = glob.glob("../*.deb") - ddeb_files = glob.glob("../*.ddeb") - - for f in deb_files: - shutil.copy(f, "/pkgdir/") - - for f in ddeb_files: - shutil.copy(f, "/pkgdir/") - - built_artifacts = { - os.path.basename(artifact) for artifact in deb_files + ddeb_files - } - - # 6. Testing - # Re-scan - os.chdir("/pkgdir") - run_cmd("dpkg-scanpackages . | xz - > /pkgdir/Packages.xz", shell=True) - - with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as f: - f.write("deb [trusted=yes] file:/pkgdir ./\n") - - run_cmd(["apt-get", "update"]) - - print(f"Installing built packages from {build_pkg_path}/..", file=sys.stderr) - - # Since we are in /pkgdir, and files were copied there, we glob current dir. - pkgdir_debs = glob.glob(f"{build_pkg_path}/../*.deb") - if pkgdir_debs: - run_cmd(["apt", "install", "-y"] + pkgdir_debs) - - # Check Binaries - # Iterate over original source locations (copied from logic `../*.deb`) - # But since we are in /pkgdir now, let's use the files we just identified. - for deb in pkgdir_debs: - # Get contents: dpkg --contents - contents = get_output(["dpkg", "--contents", deb]) - - for line in contents.splitlines(): - parts = line.split() - if len(parts) < 6: - print("failed to read package contents", file=sys.stderr) - sys.exit(3) - - # dpkg --contents output format looks like: - # drwxr-xr-x root/root 0 2023-01-01 12:00 ./usr/bin/ - fname = parts[5] - - if "bin" not in fname: - continue - - # Convert to absolute path - if fname.startswith("./"): - fname = fname[2:] - if not fname.startswith("/"): - fname = "/" + fname - - # Check if ELF executable - file_info = get_output(["file", fname]) - if "ELF" in file_info and "executable" in file_info: - print(f"checking {fname}") - try: - run_cmd(["ldd", fname]) - except subprocess.CalledProcessError: - print(f"Installed binary {fname} has linker issue") - sys.exit(2) - - # Record the successful build before removing packages that are no longer - # current for any component or architecture. - built_current_path = f"/pkgdir/{PACKAGE}@{ARCH}.built.current" - with open(built_current_path, "w") as f: - for artifact in sorted(built_artifacts): - f.write(artifact + "\n") - - remove_stale_packages("/pkgdir") - - # Ensure the local repository metadata does not refer to removed packages. - os.chdir("/pkgdir") - run_cmd("dpkg-scanpackages . | xz - > /pkgdir/Packages.xz", shell=True) +from buildlib import main, remove_stale_packages - # Finalize tag - with open(f"/pkgdir/{PACKAGE}@{ARCH}.built.tag", "w") as f: - f.write(TAG + "\n") if __name__ == "__main__": - main() + main("generic") diff --git a/packaging/ng/buildscripts/package_config.py b/packaging/ng/buildscripts/package_config.py @@ -0,0 +1,252 @@ +#!/usr/bin/env python3 + +# This file is in the public domain. + +import json +import os +import stat +import tempfile +import tomllib +from dataclasses import dataclass +from pathlib import Path + + +class ConfigError(ValueError): + pass + + +SUPPORTED_BUILDERS = {"generic", "pnpm-workspace"} + + +@dataclass(frozen=True) +class RepositoryConfig: + name: str + url: str + builder: str = "generic" + + +@dataclass(frozen=True) +class PackageConfig: + name: str + repository: str + tag: str + debian_path: str = "" + dependencies: tuple[str, ...] = () + enabled: bool = True + builder: str | None = None + + +@dataclass(frozen=True) +class PackagingConfig: + repositories: dict[str, RepositoryConfig] + packages: dict[str, PackageConfig] + + def repository_for(self, package: PackageConfig) -> RepositoryConfig: + return self.repositories[package.repository] + + def builder_for(self, package: PackageConfig) -> str: + return package.builder or self.repository_for(package).builder + + def enabled_packages(self) -> list[str]: + return sorted(name for name, package in self.packages.items() if package.enabled) + + +def _expect_table(value, description): + if not isinstance(value, dict): + raise ConfigError(f"{description} must be a table") + return value + + +def _reject_unknown(table, allowed, description): + unknown = sorted(set(table) - set(allowed)) + if unknown: + raise ConfigError(f"unknown field(s) in {description}: {', '.join(unknown)}") + + +def _required_string(table, field, description): + value = table.get(field) + if not isinstance(value, str) or not value: + raise ConfigError(f"{description}.{field} must be a non-empty string") + return value + + +def _optional_string(table, field, default, description): + value = table.get(field, default) + if not isinstance(value, str): + raise ConfigError(f"{description}.{field} must be a string") + return value + + +def parse_config(data) -> PackagingConfig: + root = _expect_table(data, "configuration") + _reject_unknown(root, {"repositories", "packages"}, "configuration") + repository_tables = _expect_table(root.get("repositories"), "repositories") + package_tables = _expect_table(root.get("packages"), "packages") + + repositories = {} + for name, raw_repository in repository_tables.items(): + description = f"repositories.{name}" + repository = _expect_table(raw_repository, description) + _reject_unknown(repository, {"url", "builder"}, description) + builder = _optional_string(repository, "builder", "generic", description) + if builder not in SUPPORTED_BUILDERS: + raise ConfigError(f"{description}.builder is not supported: {builder!r}") + repositories[name] = RepositoryConfig( + name=name, + url=_required_string(repository, "url", description), + builder=builder, + ) + + packages = {} + for name, raw_package in package_tables.items(): + description = f"packages.{name}" + package = _expect_table(raw_package, description) + _reject_unknown( + package, + { + "repository", + "tag", + "debian_path", + "dependencies", + "enabled", + "builder", + }, + description, + ) + repository = _required_string(package, "repository", description) + if repository not in repositories: + raise ConfigError(f"{description} references unknown repository {repository!r}") + dependencies = package.get("dependencies", []) + if not isinstance(dependencies, list) or not all( + isinstance(dependency, str) and dependency for dependency in dependencies + ): + raise ConfigError(f"{description}.dependencies must be an array of strings") + if len(set(dependencies)) != len(dependencies): + raise ConfigError(f"{description}.dependencies contains duplicates") + enabled = package.get("enabled", True) + if not isinstance(enabled, bool): + raise ConfigError(f"{description}.enabled must be a boolean") + builder = package.get("builder") + if builder is not None and (not isinstance(builder, str) or not builder): + raise ConfigError(f"{description}.builder must be a non-empty string") + if builder is not None and builder not in SUPPORTED_BUILDERS: + raise ConfigError(f"{description}.builder is not supported: {builder!r}") + packages[name] = PackageConfig( + name=name, + repository=repository, + tag=_required_string(package, "tag", description), + debian_path=_optional_string(package, "debian_path", "", description), + dependencies=tuple(dependencies), + enabled=enabled, + builder=builder, + ) + + for package in packages.values(): + for dependency in package.dependencies: + if dependency not in packages: + raise ConfigError( + f"packages.{package.name} references unknown dependency {dependency!r}" + ) + + _check_dependency_cycles(packages) + return PackagingConfig(repositories=repositories, packages=packages) + + +def _check_dependency_cycles(packages): + permanent = set() + temporary = [] + + def visit(name): + if name in permanent: + return + if name in temporary: + cycle = temporary[temporary.index(name) :] + [name] + raise ConfigError(f"package dependency cycle: {' -> '.join(cycle)}") + temporary.append(name) + for dependency in packages[name].dependencies: + visit(dependency) + temporary.pop() + permanent.add(name) + + for name in sorted(packages): + visit(name) + + +def load_config(path) -> PackagingConfig: + path = Path(path) + try: + with path.open("rb") as config_file: + return parse_config(tomllib.load(config_file)) + except (OSError, tomllib.TOMLDecodeError) as exc: + raise ConfigError(f"unable to read {path}: {exc}") from exc + + +def _toml_string(value): + return json.dumps(value, ensure_ascii=False) + + +def _toml_string_array(values): + return "[" + ", ".join(_toml_string(value) for value in values) + "]" + + +def serialize_config(config: PackagingConfig) -> str: + lines = [ + "# Package definitions for taler-pkg.", + "# This file is rewritten in canonical form by `taler-pkg upgrade`.", + "", + ] + for name in sorted(config.repositories): + repository = config.repositories[name] + lines.extend( + [ + f"[repositories.{_toml_string(name)}]", + f"url = {_toml_string(repository.url)}", + f"builder = {_toml_string(repository.builder)}", + "", + ] + ) + for name in sorted(config.packages): + package = config.packages[name] + lines.extend( + [ + f"[packages.{_toml_string(name)}]", + f"repository = {_toml_string(package.repository)}", + f"tag = {_toml_string(package.tag)}", + f"debian_path = {_toml_string(package.debian_path)}", + f"dependencies = {_toml_string_array(package.dependencies)}", + f"enabled = {'true' if package.enabled else 'false'}", + ] + ) + if package.builder is not None: + lines.append(f"builder = {_toml_string(package.builder)}") + lines.append("") + return "\n".join(lines) + + +def write_config(path, config: PackagingConfig): + path = Path(path) + serialized = serialize_config(config) + reparsed = parse_config(tomllib.loads(serialized)) + if reparsed != config: + raise ConfigError("serialized configuration did not round-trip") + + mode = stat.S_IMODE(path.stat().st_mode) if path.exists() else 0o644 + fd, temporary_name = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, mode) + with os.fdopen(fd, "w", encoding="utf-8") as config_file: + config_file.write(serialized) + config_file.flush() + os.fsync(config_file.fileno()) + os.replace(temporary_name, path) + directory_fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + except BaseException: + try: + os.unlink(temporary_name) + except FileNotFoundError: + pass + raise diff --git a/packaging/ng/buildscripts/pnpm-workspace b/packaging/ng/buildscripts/pnpm-workspace @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +# This file is in the public domain. + +from buildlib import main + + +if __name__ == "__main__": + main("pnpm-workspace") diff --git a/packaging/ng/packages.toml b/packaging/ng/packages.toml @@ -0,0 +1,260 @@ +# Package definitions for taler-pkg. +# This file is rewritten in canonical form by `taler-pkg upgrade`. + +[repositories."anastasis"] +url = "git://git.taler.net/anastasis.git" +builder = "generic" + +[repositories."anastasis-gtk"] +url = "git://git.taler.net/anastasis-gtk.git" +builder = "generic" + +[repositories."challenger"] +url = "git://git.taler.net/challenger.git" +builder = "generic" + +[repositories."depolymerization"] +url = "git://git.taler.net/depolymerization.git" +builder = "generic" + +[repositories."donau"] +url = "git://git.taler.net/donau.git" +builder = "generic" + +[repositories."gnunet"] +url = "git://git.gnunet.org/gnunet.git" +builder = "generic" + +[repositories."libeufin"] +url = "git://git.taler.net/libeufin.git" +builder = "generic" + +[repositories."paivana"] +url = "git://git.taler.net/paivana.git" +builder = "generic" + +[repositories."robocop"] +url = "git://git.taler.net/robocop.git" +builder = "generic" + +[repositories."sync"] +url = "git://git.taler.net/sync.git" +builder = "generic" + +[repositories."taler-challenger-helpers"] +url = "git://git.taler.net/taler-challenger-helpers.git" +builder = "generic" + +[repositories."taler-directory"] +url = "git://git.taler.net/taldir.git" +builder = "generic" + +[repositories."taler-exchange"] +url = "git://git.taler.net/exchange.git" +builder = "generic" + +[repositories."taler-mailbox"] +url = "git://git.taler.net/taler-mailbox.git" +builder = "generic" + +[repositories."taler-mdb"] +url = "git://git.taler.net/taler-mdb.git" +builder = "generic" + +[repositories."taler-merchant"] +url = "git://git.taler.net/merchant.git" +builder = "generic" + +[repositories."taler-merchant-demos"] +url = "git://git.taler.net/taler-merchant-demos" +builder = "generic" + +[repositories."taler-rust"] +url = "git://git.taler.net/taler-rust.git" +builder = "generic" + +[repositories."typescript-core"] +url = "git://git.taler.net/taler-typescript-core.git" +builder = "pnpm-workspace" + +[packages."anastasis"] +repository = "anastasis" +tag = "v0.8.1" +debian_path = "" +dependencies = ["gnunet", "taler-merchant"] +enabled = true + +[packages."anastasis-gtk"] +repository = "anastasis-gtk" +tag = "v0.8.1" +debian_path = "" +dependencies = ["anastasis"] +enabled = true + +[packages."challenger"] +repository = "challenger" +tag = "v1.6.3-dev.1" +debian_path = "" +dependencies = ["taler-exchange"] +enabled = true + +[packages."challenger-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/challenger-webui" +dependencies = [] +enabled = true + +[packages."depolymerization"] +repository = "depolymerization" +tag = "v0.0.1" +debian_path = "" +dependencies = [] +enabled = false + +[packages."donau"] +repository = "donau" +tag = "v1.6.7" +debian_path = "" +dependencies = ["gnunet", "taler-exchange"] +enabled = true + +[packages."gnunet"] +repository = "gnunet" +tag = "v0.28.1-dev.5" +debian_path = "" +dependencies = [] +enabled = true + +[packages."libeufin"] +repository = "libeufin" +tag = "v1.6.9-dev.1" +debian_path = "" +dependencies = [] +enabled = true + +[packages."libeufin-bank-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/libeufin-bank-webui" +dependencies = [] +enabled = true + +[packages."paivana-httpd"] +repository = "paivana" +tag = "v1.6.3" +debian_path = "" +dependencies = ["taler-merchant", "taler-exchange", "gnunet"] +enabled = true + +[packages."robocop"] +repository = "robocop" +tag = "v1.6.0" +debian_path = "" +dependencies = [] +enabled = true + +[packages."sync"] +repository = "sync" +tag = "v1.6.0-dev.1" +debian_path = "" +dependencies = ["taler-merchant", "taler-exchange", "gnunet"] +enabled = false + +[packages."taler-auditor-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-auditor-webui" +dependencies = [] +enabled = true + +[packages."taler-challenger-helpers"] +repository = "taler-challenger-helpers" +tag = "v1.6.2" +debian_path = "" +dependencies = [] +enabled = true + +[packages."taler-directory"] +repository = "taler-directory" +tag = "v1.5.1" +debian_path = "" +dependencies = [] +enabled = false + +[packages."taler-exchange"] +repository = "taler-exchange" +tag = "v1.6.14" +debian_path = "" +dependencies = ["gnunet"] +enabled = true + +[packages."taler-exchange-aml-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-exchange-aml-webui" +dependencies = [] +enabled = true + +[packages."taler-exchange-kyc-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-exchange-kyc-webui" +dependencies = [] +enabled = true + +[packages."taler-harness"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-harness" +dependencies = [] +enabled = true + +[packages."taler-mailbox"] +repository = "taler-mailbox" +tag = "v1.5.1" +debian_path = "" +dependencies = [] +enabled = false + +[packages."taler-mdb"] +repository = "taler-mdb" +tag = "v1.6.0" +debian_path = "" +dependencies = ["gnunet", "taler-exchange", "taler-merchant"] +enabled = false + +[packages."taler-merchant"] +repository = "taler-merchant" +tag = "v1.6.19" +debian_path = "" +dependencies = ["gnunet", "taler-exchange", "donau"] +enabled = true + +[packages."taler-merchant-demos"] +repository = "taler-merchant-demos" +tag = "v1.6.1" +debian_path = "" +dependencies = [] +enabled = false + +[packages."taler-merchant-webui"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-merchant-webui" +dependencies = [] +enabled = true + +[packages."taler-rust"] +repository = "taler-rust" +tag = "v1.5.0" +debian_path = "" +dependencies = [] +enabled = true + +[packages."taler-wallet-cli"] +repository = "typescript-core" +tag = "v1.6.33-dev.3" +debian_path = "packages/taler-wallet-cli" +dependencies = [] +enabled = true diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg @@ -10,6 +10,7 @@ import platform import os import re import sys +from dataclasses import replace from pathlib import Path # Make local util package available @@ -17,63 +18,33 @@ file = Path(__file__).resolve() parent, root = file.parent, file.parents[1] sys.path.append(str(root)) -from util import vercomp +from util import vercomp # noqa: E402 mydir = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(os.path.join(mydir, "buildscripts")) + +from package_config import ConfigError, load_config, write_config # noqa: E402 archs = ["arm64", "amd64"] host = "taler.net" native_arch = "amd64" if platform.machine().lower() in ("x86_64", "amd64") else "arm64" - -components = [ - "taler-wallet-cli", - "taler-merchant-webui", - "taler-exchange-kyc-webui", - "taler-exchange-aml-webui", - "libeufin-bank-webui", - "taler-auditor-webui", - "taler-challenger-helpers", - "challenger-webui", - "gnunet", - "libeufin", - "donau", - "paivana-httpd", - "challenger", - "taler-exchange", - "taler-harness", - "taler-merchant", - "taler-rust", - "robocop", - #"depolymerization", - # These two packages don't have good debs yet, - # Debian complains "No section given for ..., skipping. - # "taler-directory", - # "taler-mailbox", - # We don't publish packages for these yet - # "taler-mdb", - # "taler-merchant-demos", - "anastasis", - "anastasis-gtk", - # Currently not used anywhere - # "sync", -] - +config_path = Path(mydir) / "packages.toml" +config = load_config(config_path) +components = config.enabled_packages() deps = { - "taler-exchange": ["gnunet"], - "anastasis": ["gnunet", "taler-merchant"], - "anastasis-gtk": ["anastasis"], - "taler-merchant": ["gnunet", "taler-exchange", "donau"], - "donau": ["gnunet", "taler-exchange"], - "challenger": ["taler-exchange"], - # "taler-mdb": ["gnunet", "taler-exchange", "taler-merchant"], - "sync": ["taler-merchant", "taler-exchange", "gnunet"], - "paivana-httpd": ["taler-merchant", "taler-exchange", "gnunet"], + name: list(package.dependencies) + for name, package in config.packages.items() + if package.dependencies } # Compute reverse dependencies rdeps = {} for n1, d in deps.items(): + if n1 not in components: + continue for n2 in d: + if n2 not in components: + continue rd = rdeps.setdefault(n2, []) if n1 not in rd: rd.append(n1) @@ -125,7 +96,7 @@ def find_outdated(pkgdir, arch, roots): """Find outdated components based on tag files""" outdated = set() for component in roots: - ver_requested = open(f"buildconfig/{component}.tag").read().strip() + ver_requested = config.packages[component].tag built_tag_file = pkgdir / f"{component}@{arch}.built.tag" ver_built = None if built_tag_file.exists(): @@ -136,10 +107,58 @@ def find_outdated(pkgdir, arch, roots): return outdated +def component_group_key(component): + package = config.packages[component] + repository = config.repository_for(package) + builder = config.builder_for(package) + if builder == "generic": + # Generic builds are deliberately isolated even when repositories match. + return repository.url, package.tag, builder, component + return repository.url, package.tag, builder, "" + + +def group_components(selected, buildorder): + """Group selected components and topologically order the resulting groups.""" + selected = set(selected) + groups = {} + component_to_group = {} + for component in buildorder: + if component not in selected: + continue + key = component_group_key(component) + groups.setdefault(key, []).append(component) + component_to_group[component] = key + + group_dependencies = {key: set() for key in groups} + for component, key in component_to_group.items(): + for dependency in deps.get(component, []): + dependency_key = component_to_group.get(dependency) + if dependency_key is not None and dependency_key != key: + group_dependencies[key].add(dependency_key) + + ordered = [] + permanent = set() + temporary = set() + + def visit(key): + if key in permanent: + return + if key in temporary: + raise ConfigError("build groups contain a dependency cycle") + temporary.add(key) + for dependency in sorted(group_dependencies[key]): + visit(dependency) + temporary.remove(key) + permanent.add(key) + ordered.append(groups[key]) + + for key in groups: + visit(key) + return ordered + + def build(cfg): - transitive = False - if cfg.transitive: - transitive = True + transitive = cfg.transitive distro = cfg.distro vendor, codename = distro.split("-", 1) print("building", distro) @@ -193,14 +212,10 @@ def build(cfg): print("outdated closure", closure) - for component in buildorder: - if transitive: - if component not in closure: - continue - else: - if component not in outdated: - continue - print("building", component) + selected = closure if transitive else outdated + for group in group_components(selected, buildorder): + builder = config.builder_for(config.packages[group[0]]) + print("building", " ".join(group), f"with {builder}") pkgdir.mkdir(parents=True, exist_ok=True) cmd = [ "podman", @@ -224,20 +239,20 @@ def build(cfg): "--env", "CARGO_BUILD_BUILD_DIR=/root/.cargo-build", "--mount", - f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared", + f"type=bind,source={cachedir}/{distro}/apt-archives,target=/var/cache/apt/archives,relabel=shared,U=true", "--mount", - f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared", + f"type=bind,source={cachedir}/{distro}/apt-lists,target=/var/lib/apt/lists,relabel=shared,U=true", "--mount", f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly", "--mount", - f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly", + f"type=bind,source={config_path},target=/packages.toml,readonly", "--mount", f"type=bind,source={pkgdir},target=/pkgdir", image_tag, - "/buildscripts/generic", - component, + f"/buildscripts/{builder}", codename, arch, + *group, ] if not cfg.dry: subprocess.run( @@ -257,7 +272,6 @@ def promote(cfg): dry = cfg.dry distro = cfg.distro vendor, codename = distro.split("-", 1) - listfmt = "${package}_${version}_${architecture}.${$type}\n" if dry: subprocess.run( [ @@ -310,7 +324,7 @@ def test(cfg): distro = f"{vendor}-{codename}" image_tag = f"localhost/taler-packaging-{distro}:latest" dockerfile = f"distros/{distro}.Dockerfile" - cachedir = Path(f"cache").absolute() + cachedir = Path("cache").absolute() print("building base image") subprocess.run( [ @@ -379,7 +393,7 @@ def publish(cfg): elif deb.endswith(".ddeb"): pkg1, ver1, arch1 = deb.removesuffix(".ddeb").split("_") else: - raise Error(f"invalid deb filename: {deb}") + raise ValueError(f"invalid deb filename: {deb}") fresh = True server_deb = None # If the server has the same or a later version, @@ -458,72 +472,6 @@ def publish(cfg): ) -def get_remote_version(url): - """Get the latest stable tag from the git repo""" - # Construct the git command - # We use -c versionsort.suffix=- to ensure correct semantic version sorting - cmd = [ - "git", - "-c", - "versionsort.suffix=-", - "ls-remote", - "--exit-code", - "--refs", - "--sort=version:refname", - "--tags", - url, - "*.*.*", - ] - - result = subprocess.run(cmd, capture_output=True, text=True, check=True) - - # Parse the output - # Output format is usually: <hash>\trefs/tags/<tagname> - lines = result.stdout.strip().split("\n") - - valid_tags = [] - - for line in lines: - parts = line.split() - if len(parts) < 2: - continue - - # refs/tags/v1.0.0 -> v1.0.0 - ref_path = parts[1] - tag = ref_path.split("/")[-1] - - # Exclude pre-release semver versions - if tag.startswith("v") and "-" in tag: - continue - - valid_tags.append(tag) - - if valid_tags: - return valid_tags[-1] - return "(none)" - - -def check_version(name, url): - """ - Compares local buildconfig version with remote git version. - """ - ver = get_remote_version(url) - config_path = os.path.join("buildconfig", f"{name}.tag") - with open(config_path, "r") as f: - curr = f.read().strip() - prefix = "[!] " if curr != ver else "" - print(f"{prefix}{name} curr: {curr} latest: {ver}") - - -def print_latest(cfg): - """Print latest upstream tag for each component""" - for name in components: - config_path = os.path.join("buildconfig", f"{name}.giturl") - with open(config_path, "r") as f: - giturl = f.read().strip() - check_version(name, giturl) - - # Tag syntax variants supported by buildscripts/generic: # v$maj.$min.$patch => release version # v$maj.$min.$patch-dev.$n => dev version @@ -585,55 +533,67 @@ def latest_tag(tags, dev): return best +def print_latest(cfg): + """Print the latest stable upstream tag for each enabled component.""" + remote_tags = {} + for name in components: + package = config.packages[name] + url = config.repository_for(package).url + if url not in remote_tags: + remote_tags[url] = list_remote_tags(url) + latest = latest_tag(remote_tags[url], False) or "(none)" + prefix = "[!] " if package.tag != latest else "" + print(f"{prefix}{name} curr: {package.tag} latest: {latest}") + + def upgrade(cfg): - """Upgrade tag files in buildconfig to the latest upstream tag""" + """Upgrade package tags to the latest tag from their repositories.""" names = cfg.components if not names: - names = sorted(p.stem for p in Path("buildconfig").glob("*.tag")) + names = sorted(config.packages) + unknown = sorted(set(names) - set(config.packages)) + if unknown: + raise ConfigError(f"unknown package(s): {', '.join(unknown)}") # Multiple components can share a repo, only ask each remote once. remote_tags = {} upgraded = [] + updated_packages = dict(config.packages) for name in names: - tag_file = Path("buildconfig") / f"{name}.tag" - url_file = Path("buildconfig") / f"{name}.giturl" - if not url_file.exists(): - print(f"[?] {name} has no giturl, skipping", file=sys.stderr) - continue - giturl = url_file.read_text().strip() + package = config.packages[name] + giturl = config.repository_for(package).url if giturl not in remote_tags: remote_tags[giturl] = list_remote_tags(giturl) latest = latest_tag(remote_tags[giturl], cfg.dev) if latest is None: print(f"[?] {name} has no usable tag in {giturl}, skipping", file=sys.stderr) continue - curr = None - if tag_file.exists(): - curr = tag_file.read_text().strip() - currkey = tag_sortkey(curr) - if currkey is None: - print( - f"[?] {name} tag {curr} has unsupported syntax, skipping", - file=sys.stderr, - ) - continue - latestkey = tag_sortkey(latest) - if currkey > latestkey: - # Happens when the tag file pins a dev version but only - # production tags are considered. - print(f" {name} {curr} (newer than latest {latest})") - continue - if currkey == latestkey: - print(f" {name} {curr} (up to date)") - continue - print(f"[!] {name} {curr or '(none)'} -> {latest}") + curr = package.tag + currkey = tag_sortkey(curr) + if currkey is None: + print( + f"[?] {name} tag {curr} has unsupported syntax, skipping", + file=sys.stderr, + ) + continue + latestkey = tag_sortkey(latest) + if currkey > latestkey: + # Happens when a package pins a dev version but only production + # tags are considered. + print(f" {name} {curr} (newer than latest {latest})") + continue + if currkey == latestkey: + print(f" {name} {curr} (up to date)") + continue + print(f"[!] {name} {curr} -> {latest}") upgraded.append(name) if not cfg.dry: - tag_file.write_text(latest + "\n") + updated_packages[name] = replace(package, tag=latest) if not upgraded: print("nothing to upgrade") elif cfg.dry: print("would upgrade:", " ".join(upgraded)) else: + write_config(config_path, replace(config, packages=updated_packages)) print("upgraded:", " ".join(upgraded)) @@ -653,16 +613,15 @@ def main(): parser_build.add_argument( "--no-transitive", help="Do not build transitive deps of changed components (default)", - action="store_true", + action="store_false", dest="transitive", - default=None, + default=False, ) parser_build.add_argument( "--transitive", help="Build transitive deps of changed components", - action="store_false", + action="store_true", dest="transitive", - default=None, ) parser_build.add_argument( "--arch", @@ -702,7 +661,7 @@ def main(): parser_upgrade.add_argument( "components", nargs="*", - help="Components to upgrade (default: all components in buildconfig)", + help="Components to upgrade (default: all packages in packages.toml)", ) parser_upgrade.add_argument( "--dev", diff --git a/packaging/ng/testing/test_build_grouping.py b/packaging/ng/testing/test_build_grouping.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 + +# This file is in the public domain. + +import importlib.machinery +import importlib.util +import sys +import unittest +from dataclasses import replace +from io import StringIO +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +ROOT = Path(__file__).parents[1] +sys.path.insert(0, str(ROOT / "buildscripts")) +LOADER = importlib.machinery.SourceFileLoader("taler_pkg", str(ROOT / "taler-pkg")) +SPEC = importlib.util.spec_from_loader(LOADER.name, LOADER) +TALER_PKG = importlib.util.module_from_spec(SPEC) +LOADER.exec_module(TALER_PKG) + + +class BuildGroupingTests(unittest.TestCase): + def test_typescript_packages_share_one_group(self): + buildorder = TALER_PKG.buildsort(TALER_PKG.components) + typescript_packages = { + name + for name in TALER_PKG.components + if TALER_PKG.config.packages[name].repository == "typescript-core" + } + + groups = TALER_PKG.group_components(typescript_packages, buildorder) + + self.assertEqual(1, len(groups)) + self.assertEqual(typescript_packages, set(groups[0])) + + def test_different_tags_create_different_groups(self): + original = TALER_PKG.config + packages = dict(original.packages) + packages["taler-wallet-cli"] = replace( + packages["taler-wallet-cli"], tag="v9.9.9" + ) + TALER_PKG.config = replace(original, packages=packages) + try: + selected = {"taler-wallet-cli", "taler-harness"} + groups = TALER_PKG.group_components( + selected, TALER_PKG.buildsort(TALER_PKG.components) + ) + finally: + TALER_PKG.config = original + + self.assertEqual(2, len(groups)) + self.assertEqual(selected, {group[0] for group in groups}) + + def test_generic_packages_remain_isolated(self): + selected = {"anastasis", "anastasis-gtk"} + + groups = TALER_PKG.group_components( + selected, TALER_PKG.buildsort(TALER_PKG.components) + ) + + self.assertEqual([["anastasis"], ["anastasis-gtk"]], groups) + + def test_upgrade_queries_a_shared_repository_once(self): + arguments = SimpleNamespace( + components=["taler-wallet-cli", "taler-harness"], + dev=True, + dry=True, + ) + with ( + patch.object( + TALER_PKG, + "list_remote_tags", + return_value=["v1.6.33-dev.2", "v1.6.33-dev.3"], + ) as list_remote_tags, + patch("sys.stdout", new_callable=StringIO), + ): + TALER_PKG.upgrade(arguments) + + list_remote_tags.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/ng/testing/test_buildlib.py b/packaging/ng/testing/test_buildlib.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +# This file is in the public domain. + +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +BUILDSCRIPTS = Path(__file__).parents[1] / "buildscripts" +sys.path.insert(0, str(BUILDSCRIPTS)) + +import buildlib # noqa: E402 +from package_config import ConfigError, parse_config # noqa: E402 + + +class BuildLibraryTests(unittest.TestCase): + def test_pnpm_workspace_is_installed_and_built_once(self): + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + package_paths = [root / "one", root / "two"] + for number, package_path in enumerate(package_paths, start=1): + package_path.mkdir() + (package_path / "package.json").write_text( + json.dumps({"name": f"@example/package-{number}"}) + ) + + with patch.object(buildlib, "run_cmd") as run_cmd: + buildlib._prepare_pnpm_workspace(root, package_paths) + + self.assertEqual(2, run_cmd.call_count) + install, build = run_cmd.call_args_list + self.assertEqual("install", install.args[0][1]) + self.assertEqual("run", build.args[0][1]) + self.assertEqual(2, install.args[0].count("--filter")) + self.assertEqual(2, build.args[0].count("--filter")) + + def test_group_validation_requires_equal_revision_and_builder(self): + config = parse_config( + { + "repositories": { + "repo": { + "url": "git://example.test/repo.git", + "builder": "pnpm-workspace", + } + }, + "packages": { + "one": {"repository": "repo", "tag": "v1.0.0"}, + "two": {"repository": "repo", "tag": "v2.0.0"}, + }, + } + ) + + with self.assertRaisesRegex(ConfigError, "same repository, tag, and builder"): + buildlib._validate_group(config, ["one", "two"], "pnpm-workspace") + + def test_failed_validation_does_not_publish_artifacts_or_markers(self): + config = parse_config( + { + "repositories": {"repo": {"url": "git://example.test/repo.git"}}, + "packages": {"one": {"repository": "repo", "tag": "v1.0.0"}}, + } + ) + package = config.packages["one"] + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + package_path = root / "source" + output_dir = root + (package_path / "debian").mkdir(parents=True) + artifact = output_dir / "one_1.0.0_all.deb" + artifact.touch() + published = root / "published" + published.mkdir() + + with ( + patch.object(buildlib, "PKGDIR", published), + patch.object(buildlib, "run_cmd"), + patch.object( + buildlib, + "_package_artifacts", + side_effect=[set(), {artifact.resolve()}], + ), + patch.object( + buildlib, + "_check_installed_binaries", + side_effect=RuntimeError("invalid binary"), + ), + ): + with self.assertRaisesRegex(RuntimeError, "invalid binary"): + buildlib._build_package( + package, + package_path, + "v1.0.0", + "trixie", + "amd64", + prebuilt=False, + ) + + self.assertEqual([], list(published.iterdir())) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/ng/testing/test_package_config.py b/packaging/ng/testing/test_package_config.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +# This file is in the public domain. + +import sys +import tempfile +import tomllib +import unittest +import stat +from dataclasses import replace +from pathlib import Path + +BUILDSCRIPTS = Path(__file__).parents[1] / "buildscripts" +sys.path.insert(0, str(BUILDSCRIPTS)) + +from package_config import ( # noqa: E402 + ConfigError, + load_config, + parse_config, + serialize_config, + write_config, +) + + +VALID_CONFIG = { + "repositories": { + "workspace": { + "url": "git://example.test/workspace.git", + "builder": "pnpm-workspace", + } + }, + "packages": { + "application": { + "repository": "workspace", + "tag": "v1.2.3", + "debian_path": "packages/application", + "dependencies": ["library"], + "enabled": True, + }, + "library": { + "repository": "workspace", + "tag": "v1.2.2", + }, + }, +} + + +class PackageConfigTests(unittest.TestCase): + def test_defaults_and_builder_inheritance(self): + config = parse_config(VALID_CONFIG) + + library = config.packages["library"] + self.assertEqual("", library.debian_path) + self.assertEqual((), library.dependencies) + self.assertTrue(library.enabled) + self.assertEqual("pnpm-workspace", config.builder_for(library)) + + def test_package_builder_override(self): + data = { + **VALID_CONFIG, + "packages": { + **VALID_CONFIG["packages"], + "library": { + **VALID_CONFIG["packages"]["library"], + "builder": "generic", + }, + }, + } + config = parse_config(data) + + self.assertEqual("generic", config.builder_for(config.packages["library"])) + + def test_rejects_unknown_builder(self): + data = { + **VALID_CONFIG, + "repositories": { + "workspace": { + **VALID_CONFIG["repositories"]["workspace"], + "builder": "unknown", + } + }, + } + + with self.assertRaisesRegex(ConfigError, "builder is not supported"): + parse_config(data) + + def test_rejects_unknown_fields_and_references(self): + with self.assertRaisesRegex(ConfigError, "unknown field"): + parse_config({**VALID_CONFIG, "surprise": {}}) + + data = { + **VALID_CONFIG, + "packages": { + **VALID_CONFIG["packages"], + "library": { + **VALID_CONFIG["packages"]["library"], + "repository": "missing", + }, + }, + } + with self.assertRaisesRegex(ConfigError, "unknown repository"): + parse_config(data) + + def test_rejects_dependency_cycles(self): + data = { + **VALID_CONFIG, + "packages": { + **VALID_CONFIG["packages"], + "library": { + **VALID_CONFIG["packages"]["library"], + "dependencies": ["application"], + }, + }, + } + + with self.assertRaisesRegex(ConfigError, "application -> library -> application"): + parse_config(data) + + def test_serializer_is_canonical_and_round_trips(self): + config = parse_config(VALID_CONFIG) + serialized = serialize_config(config) + + self.assertEqual(config, parse_config(tomllib.loads(serialized))) + self.assertLess( + serialized.index('[packages."application"]'), + serialized.index('[packages."library"]'), + ) + self.assertEqual(serialized, serialize_config(parse_config(tomllib.loads(serialized)))) + + def test_write_config_replaces_complete_document(self): + config = parse_config(VALID_CONFIG) + updated_packages = dict(config.packages) + updated_packages["library"] = replace( + updated_packages["library"], tag="v1.2.4" + ) + updated = replace(config, packages=updated_packages) + + with tempfile.TemporaryDirectory() as temporary_directory: + path = Path(temporary_directory) / "packages.toml" + path.write_text("old contents\n") + path.chmod(0o640) + write_config(path, updated) + + self.assertEqual(updated, load_config(path)) + self.assertEqual(serialize_config(updated), path.read_text()) + self.assertEqual(0o640, stat.S_IMODE(path.stat().st_mode)) + + +if __name__ == "__main__": + unittest.main()