commit d52bee9fdf87ddcd3c269dcb4df89cd0a240f4bf
parent 38ddc169ba2d48d85095c6552e4f1c9e08680a92
Author: Florian Dold <florian@dold.me>
Date: Wed, 19 Nov 2025 17:15:58 +0100
migrate build script to python (Gemini 3)
Diffstat:
4 files changed, 264 insertions(+), 138 deletions(-)
diff --git a/packaging/ng/buildscripts/generic b/packaging/ng/buildscripts/generic
@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+# This file is in the public domain.
+# Helper script to build the latest DEB packages in the container.
+
+import os
+import sys
+import subprocess
+import glob
+import shutil
+import shlex
+
+def run_cmd(cmd, shell=False, cwd=None, env=None):
+ """Helper to run commands and exit on failure (mimicking set -e)."""
+ # If specific env vars are passed, update the current env, otherwise use current
+ command_env = os.environ.copy()
+ if env:
+ command_env.update(env)
+
+ # Flush stdout so logs appear in order
+ sys.stdout.flush()
+
+ subprocess.check_call(cmd, shell=shell, cwd=cwd, env=command_env)
+
+def get_output(cmd, shell=False, cwd=None):
+ """Helper to get command output as string."""
+ return subprocess.check_output(cmd, shell=shell, cwd=cwd, text=True).strip()
+
+def main():
+ # unset LD_LIBRARY_PATH
+ if 'LD_LIBRARY_PATH' in os.environ:
+ del os.environ['LD_LIBRARY_PATH']
+
+ # Arguments
+ if len(sys.argv) < 2:
+ print("Usage: script.py <PACKAGE> [CODENAME]", file=sys.stderr)
+ sys.exit(1)
+
+ PACKAGE = sys.argv[1]
+ CODENAME = sys.argv[2] if len(sys.argv) > 2 else ""
+
+ # Path of the debian/ folder in the repository
+ DEBIANPATH = ""
+ debpath_file = f"/buildconfig/{PACKAGE}.debpath"
+ if os.path.exists(debpath_file):
+ with open(debpath_file, 'r') as f:
+ DEBIANPATH = f.read().strip()
+
+ print(f"Building {PACKAGE} with generic build logic", file=sys.stderr)
+
+ with open(f"/buildconfig/{PACKAGE}.tag", 'r') as f:
+ TAG = f.read().strip()
+
+ # 1. Setup Local Repo
+ os.chdir("/pkgdir")
+ # Using shell=True here to easily handle the pipe logic
+ run_cmd("dpkg-scanpackages . | xz - > /pkgdir/Packages.xz", shell=True)
+
+ with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as f:
+ f.write("deb [trusted=yes] file:/pkgdir ./\n")
+
+ run_cmd(["apt-get", "update"])
+
+ # 2. Prepare Build Directory
+ if not os.path.exists("/build"):
+ os.makedirs("/build")
+ os.chdir("/build")
+
+ with open(f"/buildconfig/{PACKAGE}.giturl", 'r') as f:
+ GITURL = f.read().strip()
+
+ run_cmd(["git", "config", "--global", "advice.detachedHead", "false"])
+ run_cmd(["git", "clone", "--depth=1", f"--branch={TAG}", GITURL, PACKAGE])
+
+ build_pkg_path = os.path.join("/build", PACKAGE, DEBIANPATH)
+ os.chdir(build_pkg_path)
+
+ # Get current version
+ DEB_VERSION = get_output(["dpkg-parsechangelog", "-S", "Version"])
+ print(f"Current version of {PACKAGE}/{DEBIANPATH} is {DEB_VERSION}")
+
+ # 3. Bootstrap and Install Deps
+ os.chdir(os.path.join("/build", PACKAGE))
+ run_cmd(["./bootstrap"])
+
+ os.chdir(build_pkg_path)
+
+ # Install build-time dependencies
+ # Note: Passing the tool command as a single string because it contains spaces/options
+ tool_cmd = "apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes"
+ run_cmd(["mk-build-deps", "--install", f"--tool={tool_cmd}", "debian/control"])
+
+ # Sparse checkout hint
+ with open(".version", "w") as f:
+ f.write(f"{DEB_VERSION}\n")
+
+ # 4. Configure Environment for Build
+ deb_dbg_repo = "debian/.debhelper/"
+ os.environ["DEB_DBG_SYMBOLS_REPO"] = deb_dbg_repo
+ os.makedirs(deb_dbg_repo, exist_ok=True)
+
+ # touch command
+ open(os.path.join(deb_dbg_repo, "debian-symbols-pool"), 'a').close()
+
+ os.environ["DEB_BUILD_MAINT_OPTIONS"] = "debug"
+
+ # Add distro/codename specific version trailer
+ if CODENAME:
+ env_vars = {
+ "DEBEMAIL": "deb@taler.net",
+ "NAME": "Taler Packaging Team"
+ }
+ run_cmd([
+ "dch", "-b",
+ "--distribution", "unstable",
+ "--newversion", f"{DEB_VERSION}~{CODENAME}",
+ f"Build for {CODENAME}"
+ ], env=env_vars)
+
+ # 5. Build Package
+ run_cmd(["dpkg-buildpackage", "-rfakeroot", "-b", "-uc", "-us"])
+
+ # Copy artifacts
+ # Globs for ../*.deb and ../*.ddeb relative to current dir
+ deb_files = glob.glob("../*.deb")
+ ddeb_files = glob.glob("../*.ddeb")
+
+ for f in deb_files:
+ shutil.copy(f, "/pkgdir/")
+
+ for f in ddeb_files:
+ shutil.copy(f, "/pkgdir/")
+
+ # Save built filenames
+ built_current_path = f"/pkgdir/{PACKAGE}.built.current"
+ with open(built_current_path, 'w') as f: # 'w' mimics truncate -s0
+ for deb in deb_files:
+ f.write(os.path.basename(deb) + "\n")
+ for ddeb in ddeb_files:
+ f.write(os.path.basename(ddeb) + "\n")
+
+ # 6. Testing
+ # Re-scan
+ os.chdir("/pkgdir")
+ run_cmd("dpkg-scanpackages . | xz - > /pkgdir/Packages.xz", shell=True)
+
+ with open("/etc/apt/sources.list.d/taler-packaging-local.list", "w") as f:
+ f.write("deb [trusted=yes] file:/pkgdir ./\n")
+
+ run_cmd(["apt-get", "update"])
+
+ print(f"Installing built packages from {build_pkg_path}/..", file=sys.stderr)
+
+ # Since we are in /pkgdir, and files were copied there, we glob current dir.
+ pkgdir_debs = glob.glob(f"{build_pkg_path}/../*.deb")
+ if pkgdir_debs:
+ run_cmd(["apt", "install", "-y"] + pkgdir_debs)
+
+ # Check Binaries
+ # Iterate over original source locations (copied from logic `../*.deb`)
+ # But since we are in /pkgdir now, let's use the files we just identified.
+ for deb in pkgdir_debs:
+ # Get contents: dpkg --contents
+ contents = get_output(["dpkg", "--contents", deb])
+
+ for line in contents.splitlines():
+ parts = line.split()
+ if len(parts) < 6:
+ print("failed to read package contents", file=sys.stderr)
+ sys.exit(3)
+
+ # dpkg --contents output format looks like:
+ # drwxr-xr-x root/root 0 2023-01-01 12:00 ./usr/bin/
+ fname = parts[5]
+
+ if "bin" not in fname:
+ continue
+
+ # Convert to absolute path
+ if fname.startswith("./"):
+ fname = fname[2:]
+ if not fname.startswith("/"):
+ fname = "/" + fname
+
+ # Check if ELF executable
+ file_info = get_output(["file", fname])
+ if "ELF" in file_info and "executable" in file_info:
+ print(f"checking {fname}")
+ try:
+ run_cmd(["ldd", fname])
+ except subprocess.CalledProcessError:
+ print(f"Installed binary {fname} has linker issue")
+ sys.exit(2)
+
+ # Finalize tag
+ with open(f"/pkgdir/{PACKAGE}.built.tag", "w") as f:
+ f.write(TAG + "\n")
+
+if __name__ == "__main__":
+ main()
diff --git a/packaging/ng/buildscripts/generic.sh b/packaging/ng/buildscripts/generic.sh
@@ -1,107 +0,0 @@
-#!/bin/sh
-# This file is in the public domain.
-# Helper script to build the latest DEB packages in the container.
-
-set -eu
-unset LD_LIBRARY_PATH
-
-PACKAGE=$1
-# Path of the debian/ folder in the repository
-DEBIANPATH=""
-if [[ -e /buildconfig/$PACKAGE.debpath ]]; then
- DEBIANPATH=$(cat /buildconfig/$PACKAGE.debpath)
-fi
-
-CODENAME=${2:-}
-
-echo Building $1 with generic build logic >&2
-
-TAG=$(cat /buildconfig/$PACKAGE.tag)
-
-cd /pkgdir
-dpkg-scanpackages . | xz - >/pkgdir/Packages.xz
-echo "deb [trusted=yes] file:/pkgdir ./" >/etc/apt/sources.list.d/taler-packaging-local.list
-apt-get update
-
-mkdir -p /build
-cd /build
-
-GITURL=$(cat /buildconfig/$PACKAGE.giturl)
-
-git config --global advice.detachedHead false
-git clone --depth=1 --branch=$TAG "$GITURL" "$PACKAGE"
-
-cd "/build/$PACKAGE/$DEBIANPATH"
-
-# Get current version from debian/control file.
-DEB_VERSION=$(dpkg-parsechangelog -S Version)
-
-echo "Current version of $PACKAGE/$DEBIANPATH is $DEB_VERSION"
-
-#apt-cache show "$PACKAGE" | grep "Version: $DEB_VERSION" >/dev/null && found=true || found=false
-#if [ $found = true ]; then
-# echo "$PACKAGE version $DEB_VERSION already built, skipping"
-# exit 0
-#fi
-
-cd "/build/$PACKAGE"
-./bootstrap
-
-cd "/build/$PACKAGE/$DEBIANPATH"
-
-# Install build-time dependencies.
-mk-build-deps --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-install-recommends --yes' debian/control
-
-# We do a sparse checkout, so we need to hint
-# the version to the build system.
-echo "$DEB_VERSION" >.version
-
-export DEB_DBG_SYMBOLS_REPO="debian/.debhelper/"
-mkdir -p "${DEB_DBG_SYMBOLS_REPO}"
-touch "${DEB_DBG_SYMBOLS_REPO}/debian-symbols-pool"
-export DEB_BUILD_MAINT_OPTIONS=debug
-
-# Add distro/codename specific version trailer.
-if [[ ! -z $CODENAME ]]; then
- DEBEMAIL=deb@taler.net NAME="Taler Packaging Team" dch -b --distribution unstable --newversion "$DEB_VERSION~$CODENAME" "Build for $CODENAME"
-fi
-
-dpkg-buildpackage -rfakeroot -b -uc -us
-cp ../*.deb /pkgdir/
-cp ../*.ddeb /pkgdir/ &> /dev/null || true
-
-# We might not have .ddeb files
-shopt -s nullglob
-
-# Save current deb file names for this package.
-truncate -s0 /pkgdir/$PACKAGE.built.current
-for deb in ../*.deb; do
- echo $(basename $deb) >>/pkgdir/$PACKAGE.built.current
-done
-for deb in ../*.ddeb; do
- echo $(basename $deb) >>/pkgdir/$PACKAGE.built.current
-done
-
-# Now do some basic tests to check the packages actually work
-
-# We need to re-scan packages to account for the newly built packages.
-pushd /pkgdir
-dpkg-scanpackages . | xz - >/pkgdir/Packages.xz
-echo "deb [trusted=yes] file:/pkgdir ./" >/etc/apt/sources.list.d/taler-packaging-local.list
-apt-get update
-popd
-
-apt install -y ../*.deb
-
-for deb in ../*.deb; do
- for f in $(dpkg --contents $deb | awk '{ print $6 }' | grep "bin" | sed -e 's|^[.]/|/|'); do
- if file $f | grep -q 'ELF.*executable'; then
- echo "checking $f"
- if ! ldd "$f"; then
- echo "Installed binary $f has linker issue"
- fi
- fi
- done
-done
-
-echo $TAG >/pkgdir/$PACKAGE.built.tag
diff --git a/packaging/ng/helpers/print-latest-versions b/packaging/ng/helpers/print-latest-versions
@@ -1,26 +0,0 @@
-#!/usr/bin/bash
-
-function getver() {
- ver=$(git -c 'versionsort.suffix=-' \
- ls-remote --exit-code --refs --sort='version:refname' --tags $2 '*.*.*' \
- | grep -v 'v.*-' \
- | tail --lines=1 \
- | cut --delimiter='/' --fields=3)
- curr=$(cat buildconfig/$1.tag)
- if [[ "$curr" != "$ver" ]]; then
- echo -n "[!] "
- fi
- echo $1 "curr: $curr" latest: $ver
-}
-
-getver taler-exchange git://git.taler.net/exchange
-getver taler-merchant git://git.taler.net/merchant
-getver taler-merchant-demos git://git.taler.net/taler-merchant-demos
-getver libeufin git://git.taler.net/libeufin
-getver taler-wallet-cli git://git.taler.net/taler-typescript-core
-getver taler-harness git://git.taler.net/taler-typescript-core
-getver gnunet git://git.gnunet.org/gnunet
-getver sync git://git.taler.net/sync
-getver anastasis git://git.taler.net/anastasis
-getver anastasis-gtk git://git.taler.net/anastasis-gtk
-getver challenger git://git.taler.net/challenger
diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg
@@ -165,7 +165,7 @@ def build(cfg):
"podman",
"run",
"-it",
- "--entrypoint=/bin/bash",
+ "--entrypoint=/bin/python3",
"--security-opt",
"label=disable",
"--mount",
@@ -179,7 +179,7 @@ def build(cfg):
"--mount",
f"type=bind,source={pkgdir},target=/pkgdir",
image_tag,
- "/buildscripts/generic.sh",
+ "/buildscripts/generic",
component,
codename,
]
@@ -190,9 +190,6 @@ def build(cfg):
)
-def print_latest(cfg):
- subprocess.run(["./helpers/print-latest-versions"], check=True)
-
def show_order(cfg):
buildorder = buildsort(list(cfg.roots))
@@ -355,6 +352,69 @@ def publish(cfg):
check=True,
)
+def get_remote_version(url):
+ """Get the latest stable tag from the git repo"""
+ # Construct the git command
+ # We use -c versionsort.suffix=- to ensure correct semantic version sorting
+ cmd = [
+ "git",
+ "-c",
+ "versionsort.suffix=-",
+ "ls-remote",
+ "--exit-code",
+ "--refs",
+ "--sort=version:refname",
+ "--tags",
+ url,
+ "*.*.*",
+ ]
+
+ result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+
+ # Parse the output
+ # Output format is usually: <hash>\trefs/tags/<tagname>
+ lines = result.stdout.strip().split("\n")
+
+ valid_tags = []
+
+ for line in lines:
+ parts = line.split()
+ if len(parts) < 2:
+ continue
+
+ # refs/tags/v1.0.0 -> v1.0.0
+ ref_path = parts[1]
+ tag = ref_path.split("/")[-1]
+
+ # Exclude pre-release semver versions
+ if tag.startswith("v") and "-" in tag:
+ continue
+
+ valid_tags.append(tag)
+
+ if valid_tags:
+ return valid_tags[-1]
+ return "(none)"
+
+
+def check_version(name, url):
+ """
+ Compares local buildconfig version with remote git version.
+ """
+ ver = get_remote_version(url)
+ config_path = os.path.join("buildconfig", f"{name}.tag")
+ with open(config_path, "r") as f:
+ curr = f.read().strip()
+ prefix = "[!] " if curr != ver else ""
+ print(f"{prefix}{name} curr: {curr} latest: {ver}")
+
+def print_latest(cfg):
+ """Print latest upstream tag for each component"""
+ for name in components:
+ config_path = os.path.join("buildconfig", f"{name}.giturl")
+ with open(config_path, "r") as f:
+ giturl = f.read().strip()
+ check_version(name, giturl)
def main():
parser = argparse.ArgumentParser(
