commit edcc2376da6fd15579b34a00201e061054d14054
parent b41072458cc1898242a7e996c57580f9a87d4ee2
Author: Florian Dold <florian@dold.me>
Date: Thu, 27 Jun 2024 17:36:46 +0200
incremental build, CLI
Diffstat:
9 files changed, 204 insertions(+), 103 deletions(-)
diff --git a/packaging/ng/build.sh b/packaging/ng/build.sh
@@ -1,67 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-usage() {
- echo Usage: $0 DISTRO >&2
- exit 1
-}
-
-if [[ $# != 1 ]]; then
- usage
-fi
-
-LABEL=$1
-IMAGE_TAG=taler-packaging-$LABEL:latest
-DOCKERFILE=distros/Dockerfile.$LABEL
-PKGDIR=packages/$LABEL
-
-if [[ ! -e "$DOCKERFILE" ]]; then
- echo Need $DOCKERFILE to build $LABEL >&2
- exit 1
-fi
-
-
-function build_base() {
- echo "Building $IMAGE_TAG from $DOCKERFILE"
- # Build the base image. Usually fast because it's cached.
- podman build -t $IMAGE_TAG -f $DOCKERFILE .
-}
-
-function run() {
- SCRIPT=$1
- shift
- mkdir -p $PKGDIR
- mkdir -p cache
- podman run -it --entrypoint=/bin/bash \
- --mount type=bind,source="$(pwd)"/buildscripts,target=/buildscripts,readonly \
- --mount type=bind,source="$(pwd)"/buildconfig,target=/buildconfig,readonly \
- --mount type=bind,source="$(pwd)"/$PKGDIR,target=/pkgdir \
- $IMAGE_TAG "/buildscripts/$SCRIPT" "$@"
-}
-
-function debug() {
- mkdir -p $PKGDIR
- podman run -it --entrypoint=/bin/bash \
- --mount type=bind,source="$(pwd)"/buildscripts,target=/buildscripts,readonly \
- --mount type=bind,source="$(pwd)"/buildconfig,target=/buildconfig,readonly \
- --mount type=bind,source="$(pwd)"/$PKGDIR,target=/pkgdir \
- $IMAGE_TAG -i
-}
-
-function build_all() {
- run generic.sh gnunet
- run generic.sh gnunet-gtk
- run generic.sh taler-exchange
- run generic.sh taler-merchant
- run generic.sh sync
- run generic.sh anastasis
- run generic.sh anastasis-gtk
- run generic.sh libeufin
- run generic.sh taler-merchant-demos
- run generic.sh taler-wallet-cli packages/taler-wallet-cli
- run generic.sh taler-harness packages/taler-harness
- run generic.sh libeufin
-}
-
-build_all
diff --git a/packaging/ng/buildconfig/README b/packaging/ng/buildconfig/README
@@ -1,5 +0,0 @@
-These files determine the git tag from which the respective components are
-built in the base Docker image.
-
-They are in separate files to make modification checking with
-staged Docker builds work nicely.
diff --git a/packaging/ng/buildconfig/taler-harness.debpath b/packaging/ng/buildconfig/taler-harness.debpath
@@ -0,0 +1 @@
+packages/taler-harness
diff --git a/packaging/ng/buildconfig/taler-wallet-cli.debpath b/packaging/ng/buildconfig/taler-wallet-cli.debpath
@@ -0,0 +1 @@
+packages/taler-wallet-cli
diff --git a/packaging/ng/buildscripts/generic.sh b/packaging/ng/buildscripts/generic.sh
@@ -7,19 +7,36 @@ unset LD_LIBRARY_PATH
PACKAGE=$1
# Path of the debian/ folder in the repository
-DEBIANPATH=${2:-.}
+DEBIANPATH=""
+if [[ -e /buildconfig/$PACKAGE.debpath ]]; then
+ DEBIANPATH=$(cat /buildconfig/$PACKAGE.debpath)
+fi
echo Building $1 with generic build logic >&2
+if [ -e /pkgdir/$PACKAGE.built.tag ]; then
+ BUILT_TAG="$(cat /pkgdir/$PACKAGE.built.tag)"
+else
+ BUILT_TAG=""
+fi
+
+TAG=$(cat /buildconfig/$PACKAGE.tag)
+
+if [[ $BUILT_TAG = $TAG ]]; then
+ echo Tag $TAG already built, skipping
+ exit 0
+fi
+
+echo Building $TAG, old tag ${BUILT_TAG:-<none>}
+
cd /pkgdir
-dpkg-scanpackages . | xz - > /pkgdir/Packages.xz
+dpkg-scanpackages . | xz - >/pkgdir/Packages.xz
echo "deb [trusted=yes] file:/pkgdir ./" >/etc/apt/sources.list.d/taler-packaging-local.list
apt-get update
mkdir -p /build
cd /build
-TAG=$(cat /buildconfig/$PACKAGE.tag)
GITURL=$(cat /buildconfig/$PACKAGE.giturl)
git config --global advice.detachedHead false
@@ -32,11 +49,11 @@ DEB_VERSION=$(dpkg-parsechangelog -S Version)
echo "Current version of $PACKAGE/$DEBIANPATH is $DEB_VERSION"
-apt-cache show "$PACKAGE" | grep "Version: $DEB_VERSION" >/dev/null && found=true || found=false
-if [ $found = true ]; then
- echo "$PACKAGE version $DEB_VERSION already built, skipping"
- exit 0
-fi
+#apt-cache show "$PACKAGE" | grep "Version: $DEB_VERSION" >/dev/null && found=true || found=false
+#if [ $found = true ]; then
+# echo "$PACKAGE version $DEB_VERSION already built, skipping"
+# exit 0
+#fi
cd "/build/$PACKAGE"
./bootstrap
@@ -48,7 +65,9 @@ mk-build-deps --install --tool='apt-get -o Debug::pkgProblemResolver=yes --no-in
# We do a sparse checkout, so we need to hint
# the version to the build system.
-echo $DEB_VERSION > .version
+echo $DEB_VERSION >.version
dpkg-buildpackage -rfakeroot -b -uc -us
cp ../*.deb /pkgdir/
+
+echo $TAG >/pkgdir/$PACKAGE.built.tag
diff --git a/packaging/ng/distros/Dockerfile.ubuntu-noble b/packaging/ng/distros/Dockerfile.ubuntu-noble
@@ -0,0 +1,30 @@
+FROM ubuntu:noble
+# This file is in the public domain.
+#
+# Docker image to build Ubuntu packages of
+# GNUnet, GNU Taler and GNU Anastasis.
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install dependencies
+RUN apt-get update
+RUN apt-get -y upgrade
+RUN apt-get -y install build-essential zip jq python3 python3-pip nodejs npm
+RUN apt-get -y install autoconf automake gcc make libtool libltdl-dev libmicrohttpd-dev libpq-dev libsqlite3-dev libunistring-dev libqrencode-dev libgcrypt-dev libsodium-dev libargon2-dev libjansson-dev recutils libgmp-dev texinfo pkgconf zlib1g-dev libopus-dev libextractor-dev libnss3-dev libcurl4-gnutls-dev autopoint
+RUN apt-get -y install libzbar-dev libmysqlclient-dev mandoc libpulse-dev libgstreamer1.0-dev libgstreamer-plugins-good1.0-dev libbluetooth-dev iptables miniupnpc libpng-dev
+RUN apt-get -y install python3-jinja2 doxygen libjose-dev iproute2 sudo
+RUN apt-get -y install wget zile
+RUN apt-get -y install libogg-dev gettext net-tools po-debconf debhelper-compat dbconfig-pgsql nginx
+RUN apt-get -y install libgtk-3-dev libgladeui-dev libmagic-dev policykit-1
+RUN apt-get -y install dbconfig-no-thanks
+RUN apt-get -y install devscripts equivs
+# For libeufin:
+RUN apt-get -y install openjdk-17-jdk python3-click python3-requests python3
+
+RUN pip install --break-system-packages sphinx_rtd_theme
+RUN npm install -g npm
+RUN /usr/local/bin/npm install -g npm pnpm node
+
+RUN apt-get update
+RUN apt-get -y upgrade
+RUN apt-get -y dist-upgrade
diff --git a/packaging/ng/helpers/print-latest-versions b/packaging/ng/helpers/print-latest-versions
@@ -0,0 +1,25 @@
+#!/usr/bin/bash
+
+function getver() {
+ ver=$(git -c 'versionsort.suffix=-' \
+ ls-remote --exit-code --refs --sort='version:refname' --tags $2 '*.*.*' \
+ | tail --lines=1 \
+ | cut --delimiter='/' --fields=3)
+ curr=$(cat buildconfig/$1.tag)
+ if [[ "$curr" != "$ver" ]]; then
+ echo -n "[!] "
+ fi
+ echo $1 "curr: $curr" latest: $ver
+}
+
+getver taler-exchange git://git.taler.net/exchange
+getver taler-merchant git://git.taler.net/merchant
+getver taler-merchant-demos git://git.taler.net/taler-merchant-demos
+getver libeufin git://git.taler.net/libeufin
+getver taler-wallet-cli git://git.taler.net/wallet-core
+getver taler-harness git://git.taler.net/wallet-core
+getver gnunet git://git.gnunet.org/gnunet
+getver gnunet-gtk git://git.gnunet.org/gnunet-gtk
+getver sync git://git.taler.net/sync
+getver anastasis git://git.taler.net/anastasis
+getver anastasis-gtk git://git.taler.net/anastasis-gtk
diff --git a/packaging/ng/print-latest-versions b/packaging/ng/print-latest-versions
@@ -1,22 +0,0 @@
-#!/usr/bin/bash
-
-function getver() {
- ver=$(git -c 'versionsort.suffix=-' \
- ls-remote --exit-code --refs --sort='version:refname' --tags $2 '*.*.*' \
- | tail --lines=1 \
- | cut --delimiter='/' --fields=3)
- curr=$(cat buildconfig/$1.tag)
- if [[ "$curr" != "$ver" ]]; then
- echo -n "[!] "
- fi
- echo $1 "curr: $curr" latest: $ver
-}
-
-getver exchange git://git.taler.net/exchange
-getver merchant git://git.taler.net/merchant
-getver merchant-demos git://git.taler.net/taler-merchant-demos
-getver libeufin git://git.taler.net/libeufin
-getver wallet git://git.taler.net/wallet-core
-getver gnunet git://git.gnunet.org/gnunet
-getver sync git://git.taler.net/sync
-getver libmhd git://git.gnunet.org/libmicrohttpd
diff --git a/packaging/ng/taler-pkg b/packaging/ng/taler-pkg
@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+from dataclasses import dataclass
+import os
+from pathlib import Path
+
+mydir = os.path.dirname(os.path.realpath(__file__))
+
+components = [
+ "anastasis",
+ "anastasis-gtk",
+ "gnunet",
+ "gnunet-gtk",
+ "libeufin",
+ "sync",
+ "taler-exchange",
+ "taler-harness",
+ "taler-merchant",
+ "taler-merchant-demos",
+ "taler-wallet-cli",
+]
+
+deps = {
+ "taler-exchange": ["gnunet"],
+ "anastasis": ["gnunet", "taler-merchant"],
+ "anastasis-gtk": ["anastasis"],
+ "taler-merchant": ["gnunet", "taler-exchange"],
+ "sync": ["taler-merchant", "taler-exchange", "gnunet"],
+ "gnunet-gtk": ["gnunet"],
+}
+
+
+def toposort(roots):
+ """Toposort components based on deps"""
+ out = []
+ stack = list(roots)
+ pmark = set()
+ tmark = set()
+ while len(stack):
+ node = stack[-1]
+ if node in pmark:
+ stack.pop()
+ tmark.discard(node)
+ continue
+ done = True
+ for dep in deps.get(node, []):
+ if dep not in pmark:
+ if dep in tmark:
+ raise Exception("cycle")
+ stack.append(dep)
+ tmark.add(node)
+ done = False
+ if done:
+ pmark.add(node)
+ out.append(node)
+ stack.pop()
+ tmark.discard(node)
+ return out
+
+
+def build(cfg):
+ distro = cfg.distro
+ print("building", distro)
+ dockerfile = f"distros/Dockerfile.{distro}"
+ image_tag = f"taler-packaging-{distro}:latest"
+ pkgdir = Path(f"packages/{distro}").absolute()
+ subprocess.run(["podman", "build", "-t", image_tag, "-f", dockerfile], check=True)
+
+ buildorder = toposort(components)
+
+ for component in buildorder:
+ print("building", component)
+ pkgdir.mkdir(parents=True, exist_ok=True)
+ subprocess.run(
+ [
+ "podman",
+ "run",
+ "-it",
+ "--entrypoint=/bin/bash",
+ "--security-opt",
+ "label=disable",
+ "--mount",
+ f"type=bind,source={mydir}/buildscripts,target=/buildscripts,readonly",
+ "--mount",
+ f"type=bind,source={mydir}/buildconfig,target=/buildconfig,readonly",
+ "--mount",
+ f"type=bind,source={pkgdir},target=/pkgdir",
+ image_tag,
+ "/buildscripts/generic.sh",
+ component,
+ ],
+ check=True,
+ )
+
+def print_latest(cfg):
+ subprocess.run(["./helpers/print-latest-versions"], check=True)
+
+
+parser = argparse.ArgumentParser(prog="taler-pkg", description="Taler Packaging Helper")
+
+subparsers = parser.add_subparsers(help="subcommands", metavar="<command>")
+
+parser_build = subparsers.add_parser("build", help="Build packages")
+parser_build.set_defaults(func=build)
+parser_build.add_argument("distro")
+
+parser_show_latest = subparsers.add_parser(
+ "show-latest", help="Show latest version of packages"
+)
+parser_show_latest.set_defaults(func=print_latest)
+
+args = parser.parse_args()
+
+if "func" not in args:
+ parser.print_help()
+else:
+ args.func(args)
