paivana

test_reverse_proxy.sh (102613B)

---

 #!/bin/bash
 #
 # Reverse-proxy integration tests for paivana.
 #
 # Starts upstream HTTP servers (one per language: C/MHD, Go, Python, Rust)
 # and a paivana-httpd instance running with -n (paywall disabled), then
 # exercises the reverse-proxy behaviors with curl, wget, and a custom
 # libcurl / raw-socket pipelining client.
 #
 # Progress markers: each test prints "<description> " with no newline,
 # then "OK" on pass or "FAIL: <detail>" on failure.  Failure exits
 # non-zero, which make(1) treats as a TEST FAILURE.
 #
 # Environment variables honored:
 #   PAIVANA_HTTPD   path to paivana-httpd binary (default: built
 #                   in the sibling src/backend tree)
 #   SRCDIR          source dir containing .py / .rs / .go sources
 #                   (default: directory of this script)
 #   BUILDDIR        directory holding upstream_mhd, upstream_go,
 #                   upstream_rs, pipeline_client (default: $PWD)
 #   KEEP_TMP=1      keep the scratch dir and log files after exit
 #   PAIVANA_PORT_BASE
 #                   first port of the ten the suite binds (default
 #                   18400); move it to run two checkouts at once
 #
 set -u
 
 function die() {
     echo "FAIL: $*" >&2
     exit 1
 }
 
 function msg() {
     printf '%s ' "$*"
 }
 
 function ok() {
     echo "OK"
 }
 
 function fail() {
     echo "FAIL: $*"
     dump_logs
     exit 1
 }
 
 function here() {
     cd -- "$(dirname -- "$0")" && pwd
 }
 
 SRCDIR="${SRCDIR:-$(here)}"
 BUILDDIR="${BUILDDIR:-$PWD}"
 
 # Default to the in-tree build path.
 PAIVANA_HTTPD="${PAIVANA_HTTPD:-$BUILDDIR/../backend/paivana-httpd}"
 if [ ! -x "$PAIVANA_HTTPD" ];
 then
     # Try the source layout (e.g. when tests are run from source tree)
     alt="$SRCDIR/../backend/paivana-httpd"
     if [ -x "$alt" ];
     then
         PAIVANA_HTTPD="$alt"
     fi
 fi
 
 if [ ! -x "$PAIVANA_HTTPD" ];
 then
     echo "SKIP: paivana-httpd binary not found (looked at $PAIVANA_HTTPD)" >&2
     exit 77
 fi
 
 # Binaries/commands for upstreams
 UPSTREAM_MHD="$BUILDDIR/upstream_mhd"
 UPSTREAM_GO="$BUILDDIR/upstream_go"
 UPSTREAM_RS="$BUILDDIR/upstream_rs"
 PIPELINE_CLIENT="$BUILDDIR/pipeline_client"
 EARLY_RESPONSE_UPSTREAM="$BUILDDIR/early_response_upstream"
 
 # Ports.  Every one of them is an offset off a single base so that the
 # whole block can be moved out of the way: two checkouts of this repo
 # (or two CI jobs on one machine) running the suite at once would
 # otherwise fight over the same ten fixed numbers, and the loser reads
 # as a paivana bug rather than as a collision.  The default keeps the
 # historical numbering.  require_ports_free() below refuses to run at
 # all when one of them is taken.
 PORT_BASE="${PAIVANA_PORT_BASE:-18400}"
 MHD_PORT=$((PORT_BASE + 1))
 GO_PORT=$((PORT_BASE + 2))
 PY_PORT=$((PORT_BASE + 3))
 RS_PORT=$((PORT_BASE + 4))
 EARLY_PORT=$((PORT_BASE + 5))
 NODRAIN_PORT=$((PORT_BASE + 6))
 TRUNC_PORT=$((PORT_BASE + 7))
 STREAM_PORT=$((PORT_BASE + 8))
 DEAD_PORT=$((PORT_BASE + 99))    # nothing may be listening here
 PAIVANA_PORT=$((PORT_BASE + 100))
 
 # NOT named TMPDIR.  That is the standard variable every child process
 # reads for its own temporary files, bash keeps the export attribute an
 # inherited TMPDIR came with, and cleanup() rm -rf's this directory
 # while paivana, the upstreams and curl may still be running out of it.
 SCRATCH="$(mktemp -d -t paivana-tests.XXXXXX)"
 LOGDIR="$SCRATCH/logs"
 mkdir -p "$LOGDIR"
 
 # paivana normally resolves its config.d via the install prefix.
 # Tests run against the uninstalled build tree, so point the
 # project's base-config override at an empty directory: no
 # auxiliary config snippets are needed for reverse-proxy tests.
 BASE_CONFIG_DIR="$SCRATCH/configd"
 mkdir -p "$BASE_CONFIG_DIR"
 export PAIVANA_BASE_CONFIG="$BASE_CONFIG_DIR"
 
 PIDS=()
 PAIVANA_PID=""
 # DESTINATION_BASE_URL the running paivana was configured with; the
 # battery derives from it the Host header paivana should be sending
 # upstream.
 PAIVANA_DEST=""
 # Path of the listening socket when paivana was started by
 # start_paivana_unix(); empty for the TCP cases.
 PAIVANA_SOCK=""
 
 function dump_logs() {
     echo "-- logs in $LOGDIR --" >&2
     for f in "$LOGDIR"/*.log; do
         [ -e "$f" ] || continue
         echo "==> $f <==" >&2
         tail -n 40 "$f" >&2
     done
 }
 
 function cleanup() {
     set +e
     for p in "${PIDS[@]:-}";
     do
         [ -n "$p" ] && kill -TERM "$p" 2>/dev/null
     done
     [ -n "$PAIVANA_PID" ] && kill -TERM "$PAIVANA_PID" 2>/dev/null
     # Give them a moment to exit cleanly
     sleep 0.2
     for p in "${PIDS[@]:-}";
     do
         [ -n "$p" ] && kill -KILL "$p" 2>/dev/null
     done
     [ -n "$PAIVANA_PID" ] && kill -KILL "$PAIVANA_PID" 2>/dev/null
     if [ "${KEEP_TMP:-0}" = "1" ];
     then
         echo "Temp files kept in $SCRATCH" >&2
     else
         rm -rf "$SCRATCH"
     fi
 }
 trap cleanup EXIT
 trap 'echo "FAIL: interrupted" >&2; exit 1' INT TERM
 
 # Does a TCP connect to the given port fail?  Used both as the
 # "nothing is squatting here" precondition and, negated, as the
 # readiness probe.
 #
 # NOTE: the /dev/tcp probe must run in a subshell — `exec` on the
 # parent shell with a failing redirection would terminate bash in
 # non-interactive mode (the 2>/dev/null does not suppress that).
 function port_is_free() {
     local host="$1" port="$2"
 
     if ( exec 7<>"/dev/tcp/$host/$port" ) 2>/dev/null;
     then
         return 1
     fi
     return 0
 }
 
 function require_ports_free() {
     # The suite's own documentation promised this and nothing did it.
     # A stranger on PAIVANA_PORT is the damaging case: our paivana dies
     # of EADDRINUSE, the readiness probe below sees the squatter accept
     # and reports "started", and every check then runs against the
     # wrong process -- with a stale paivana of a different vintage the
     # checks even pass.  DEAD_PORT has to be free for the mirror-image
     # reason: test_upstream_down asserts that connecting to it fails.
     #
     # This is an environment problem rather than a regression, so it is
     # a skip (meson reads 77 as SKIP), not a failure.
     local busy=""
 
     for p in "$@";
     do
         port_is_free 127.0.0.1 "$p" || busy="$busy $p"
     done
     if [ -n "$busy" ];
     then
         echo "SKIP: port(s) already in use:$busy" >&2
         echo "Another copy of this suite, or a stale paivana-httpd, is" \
              "holding them; re-run with PAIVANA_PORT_BASE set to a free" \
              "block of 101 ports (current base: $PORT_BASE)." >&2
         exit 77
     fi
 }
 
 function wait_for_port() {
     # Block until a TCP port accepts a connection (max ~5s), giving up
     # the moment the process that was supposed to bind it is gone.
     #
     # Without the liveness check this only asks "is *something*
     # listening", which is not the same question: a child that died of
     # EADDRINUSE (or of a config it refused) reads as started, and the
     # caller happily runs its checks against whoever holds the port.
     # It is also what made a refused startup cost the full 5 s of
     # retries instead of the milliseconds the child actually took to
     # exit -- seven of those were most of the suite's wall-clock.
     #
     # $3 is optional so that a caller with no pid to offer (a helper
     # started by some other means) still works, just without either
     # benefit.
     local host="$1" port="$2" pid="${3:-}" tries=50
 
     while [ "$tries" -gt 0 ];
     do
         if ! port_is_free "$host" "$port";
         then
             return 0
         fi
         # Order matters: probe first, so that "the port is up" always
         # wins over "the pid we were given is gone" -- a child that
         # handed the listening socket on and exited is still a service
         # that came up.
         if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null;
         then
             return 1
         fi
         sleep 0.1
         tries=$((tries - 1))
     done
     return 1
 }
 
 # Start a background upstream; record pid in PIDS.
 function start_bg() {
     local name="$1" port="$2"; shift 2
     local log="$LOGDIR/$name.log"
     ( exec "$@" "$port" ) >"$log" 2>&1 &
     local pid=$!
     PIDS+=("$pid")
     if ! wait_for_port 127.0.0.1 "$port" "$pid";
     then
         echo "FAIL: $name did not start on port $port" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 }
 
 function start_paivana() {
     # $1 = upstream base URL; any further arguments are passed to
     # paivana-httpd verbatim.  Note the shift: quoting "$@" (rather
     # than a single "$flags" string) is what keeps a caller that
     # passes no extra flags from handing paivana an empty argument.
     local dest="$1"; shift
     PAIVANA_DEST="$dest"
     local cfg="$SCRATCH/paivana.conf"
     sed -e "s|@DEST@|$dest|g" -e "s|@PORT@|$PAIVANA_PORT|g" \
         "$SRCDIR/test_reverse_proxy.conf.in" > "$cfg"
     local log="$LOGDIR/paivana.log"
     ( exec "$PAIVANA_HTTPD" -c "$cfg" -n -L WARNING "$@" ) >"$log" 2>&1 &
     PAIVANA_PID=$!
     if ! wait_for_port 127.0.0.1 "$PAIVANA_PORT" "$PAIVANA_PID";
     then
         echo "FAIL: paivana-httpd did not start on port $PAIVANA_PORT" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 }
 
 # Start the TCP daemon with extra configuration lines.  BIND_TO keeps this on
 # one MHD daemon so small connection-limit tests exercise the process-wide
 # arithmetic rather than dividing their tiny budget across IPv4 and IPv6.
 function start_paivana_with_config() {
     local dest="$1" extra="$2" level="${3:-WARNING}"
     PAIVANA_DEST="$dest"
     local cfg="$SCRATCH/paivana-custom.conf"
     sed -e "s|@DEST@|$dest|g" -e "s|@PORT@|$PAIVANA_PORT|g" \
         "$SRCDIR/test_reverse_proxy.conf.in" > "$cfg"
     printf 'BIND_TO = 127.0.0.1\n%s\n' "$extra" >> "$cfg"
     local log="$LOGDIR/paivana-custom.log"
     ( exec "$PAIVANA_HTTPD" -c "$cfg" -n -L "$level" ) >"$log" 2>&1 &
     PAIVANA_PID=$!
     if ! wait_for_port 127.0.0.1 "$PAIVANA_PORT" "$PAIVANA_PID";
     then
         echo "FAIL: custom paivana-httpd did not start" >&2
         tail -n 40 "$log" >&2
         exit 1
     fi
 }
 
 function wait_for_unix_socket() {
     # Block until the given path exists and is a socket (max ~5s), or
     # until the process that was to create it is gone.  Same reasoning
     # as wait_for_port: a leftover socket file from an earlier run is
     # the Unix-domain spelling of a squatter on the port.
     local path="$1" pid="${2:-}" tries=50
 
     while [ "$tries" -gt 0 ];
     do
         [ -S "$path" ] && return 0
         if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null;
         then
             return 1
         fi
         sleep 0.1
         tries=$((tries - 1))
     done
     return 1
 }
 
 function start_paivana_unix() {
     # Like start_paivana, but listening on a Unix socket rather than
     # TCP -- the shape the shipped packaging deploys.  A Unix peer has
     # no address, which is exactly what makes it worth testing.
     # $1 = upstream base URL; further arguments go to paivana-httpd.
     local dest="$1"; shift
     PAIVANA_DEST="$dest"
     local cfg="$SCRATCH/paivana-unix.conf"
     PAIVANA_SOCK="$SCRATCH/paivana.sock"
     rm -f "$PAIVANA_SOCK"
     sed -e "s|@DEST@|$dest|g" -e "s|@UNIXPATH@|$PAIVANA_SOCK|g" \
         "$SRCDIR/test_reverse_proxy_unix.conf.in" > "$cfg"
     local log="$LOGDIR/paivana-unix.log"
     ( exec "$PAIVANA_HTTPD" -c "$cfg" -n -L WARNING "$@" ) >"$log" 2>&1 &
     PAIVANA_PID=$!
     if ! wait_for_unix_socket "$PAIVANA_SOCK" "$PAIVANA_PID";
     then
         echo "FAIL: paivana-httpd did not create $PAIVANA_SOCK" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 }
 
 function stop_paivana() {
     if [ -n "$PAIVANA_PID" ];
     then
         kill -TERM "$PAIVANA_PID" 2>/dev/null
         wait "$PAIVANA_PID" 2>/dev/null
         PAIVANA_PID=""
     fi
 }
 
 # Start all upstreams that we have binaries for.
 function start_upstreams() {
     start_bg mhd "$MHD_PORT" "$UPSTREAM_MHD"
     if [ -x "$UPSTREAM_GO" ];
     then
         start_bg go "$GO_PORT" "$UPSTREAM_GO"
     else
         echo "NOTE: upstream_go not built, skipping Go upstream tests" >&2
         GO_PORT=""
     fi
     if [ -x "$UPSTREAM_RS" ];
     then
         start_bg rs "$RS_PORT" "$UPSTREAM_RS"
     else
         echo "NOTE: upstream_rs not built, skipping Rust upstream tests" >&2
         RS_PORT=""
     fi
     if command -v python3 >/dev/null 2>&1;
     then
         local log="$LOGDIR/py.log"
         ( exec python3 "$SRCDIR/upstream_py.py" "$PY_PORT" ) >"$log" 2>&1 &
         local pypid=$!
         PIDS+=("$pypid")
         if ! wait_for_port 127.0.0.1 "$PY_PORT" "$pypid";
         then
             echo "FAIL: upstream_py did not start on port $PY_PORT" >&2
             tail -n 20 "$log" >&2
             exit 1
         fi
     else
         echo "NOTE: python3 not available, skipping Python upstream tests" >&2
         PY_PORT=""
     fi
 }
 
 ######################################################################
 # Test helpers
 ######################################################################
 
 PAIVANA_URL() { echo "http://127.0.0.1:$PAIVANA_PORT$1"; }
 
 # GET via curl; verifies status code and a substring of the body.
 function test_get() {
     local desc="$1" path="$2" want_status="$3" want_sub="$4"
     msg "$desc"
     local out status
     out="$(curl -sS -o "$SCRATCH/body" -w '%{http_code}' "$(PAIVANA_URL "$path")" 2>"$SCRATCH/err")" \
         || { fail "curl: $(cat "$SCRATCH/err")"; }
     status="$out"
     [ "$status" = "$want_status" ] || fail "status=$status want=$want_status"
     # -F: $want_sub is a substring, not a pattern.  Without it a '.'
     # in an expected body matches anything and the check passes on a
     # body it should have rejected.
     if [ -n "$want_sub" ] && ! grep -qF -- "$want_sub" "$SCRATCH/body";
     then
         fail "body missing substring '$want_sub' (got: $(tr -d '\n' <"$SCRATCH/body" | head -c 120))"
     fi
     ok
 }
 
 # HEAD.  Status only; the "no content" half is test_head_no_body().
 function test_head() {
     local desc="$1" path="$2" want_status="$3"
     msg "$desc"
     local status
     status="$(curl -sS -I -o /dev/null -w '%{http_code}' "$(PAIVANA_URL "$path")" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "$want_status" ] || fail "status=$status want=$want_status"
     ok
 }
 
 # RFC 9110 section 9.3.2's one normative requirement on HEAD -- "the
 # server MUST NOT send content in the response" -- read off the wire
 # rather than through curl, which discards a body a HEAD response has
 # no business carrying and would therefore report the bug as a pass.
 # The path is one that produces 128 KiB under GET, so a proxy that
 # forgot the method has something to leak.
 function test_head_no_body() {
     local label="$1"
     msg "[$label] HEAD /large/131072 carries no content (RFC 9110 9.3.2)"
     raw_head "127.0.0.1" "$PAIVANA_PORT" "$SCRATCH/head_raw"
     case "$(head -c 12 "$SCRATCH/head_raw")" in
         'HTTP/1.1 200'*) ;;
         *) fail "expected 200 status line, got: $(head -c 80 "$SCRATCH/head_raw")";;
     esac
     # Bytes after the first empty line.  Zero for a conforming HEAD
     # response; anything else is content that must not be there.
     local extra
     extra="$(tr -d '\r' <"$SCRATCH/head_raw" \
              | awk 'seen { n += length ($0) + 1 }
                     /^$/ && ! seen { seen = 1 }
                     END { print n + 0 }')"
     [ "$extra" = "0" ] || \
         fail "HEAD response carried $extra bytes of content"
     ok
 }
 
 # Raw HEAD /large/131072 against $1:$2, whole response into $3.  Raw
 # rather than curl, which discards a body a HEAD response has no
 # business carrying and would report the bug as a pass.
 function raw_head() {
     local host="$1" port="$2" out="$3"
 
     ( exec 3<>"/dev/tcp/$host/$port"
       printf 'HEAD /large/131072 HTTP/1.1\r\nHost: %s:%s\r\nConnection: close\r\n\r\n' \
              "$host" "$port" >&3
       timeout 10 cat <&3 >"$out" ) 2>"$SCRATCH/err" \
         || fail "raw HEAD to $host:$port failed: $(cat "$SCRATCH/err")"
 }
 
 # Write @2 with @1 bytes of the 'A'..'Z' cycle every upstream serves
 # from /large/N, so that the response can be compared byte for byte
 # rather than merely counted.  The chunk stays a whole number of
 # 26-byte cycles, which is what keeps it aligned when repeated.
 function make_large_pattern() {
     local want="$1" out="$2"
     local chunk='ABCDEFGHIJKLMNOPQRSTUVWXYZ'
     local n i
 
     while [ "${#chunk}" -lt 65536 ];
     do
         chunk="$chunk$chunk"
     done
     n=$(( (want + ${#chunk} - 1) / ${#chunk} ))
     for ((i = 0; i < n; i++));
     do
         printf '%s' "$chunk"
     done | head -c "$want" >"$out"
 }
 
 # Generic method with optional body; checks status and body substring.
 function test_method() {
     local desc="$1" method="$2" path="$3" body="$4" want_status="$5" want_sub="$6"
     msg "$desc"
     local args=(-sS -o "$SCRATCH/body" -w '%{http_code}' -X "$method" "$(PAIVANA_URL "$path")")
     if [ -n "$body" ];
     then
         args+=(--data-binary "@$body")
     fi
     local status
     status="$(curl "${args[@]}" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "$want_status" ] || \
         fail "status=$status want=$want_status; body=$(head -c 200 "$SCRATCH/body")"
     if [ -n "$want_sub" ] && ! grep -qF -- "$want_sub" "$SCRATCH/body";
     then
         fail "body missing substring '$want_sub' (got: $(head -c 200 "$SCRATCH/body"))"
     fi
     ok
 }
 
 ######################################################################
 # Test battery — runs against whichever upstream we've pointed
 # paivana at.
 ######################################################################
 
 function run_battery() {
     local label="$1"
 
     test_get "[$label] GET /hello (basic proxy pass-through)" \
         /hello 200 "Hello from"
 
     test_get "[$label] GET /status/201 (2xx status forwarding)" \
         /status/201 201 "status 201"
 
     test_get "[$label] GET /status/404 (4xx status forwarding)" \
         /status/404 404 "status 404"
 
     test_get "[$label] GET /status/500 (5xx status forwarding)" \
         /status/500 500 "status 500"
 
     test_head "[$label] HEAD /hello" /hello 200
     test_head_no_body "$label"
 
     # 128 KiB response body, compared byte for byte against the
     # 'A'..'Z' cycle every upstream generates.  A length check alone
     # would pass on a body that arrived complete but scrambled --
     # a chunk delivered twice and another dropped, say -- which is
     # precisely the failure a buffering proxy has to be shown not to
     # have.
     test_get "[$label] GET /large/131072 (128 KiB response)" \
         /large/131072 200 ""
     msg "[$label] 128 KiB response body is byte-for-byte intact"
     local sz
     sz="$(wc -c <"$SCRATCH/body" | tr -d ' ')"
     [ "$sz" = "131072" ] || fail "got $sz bytes, expected 131072"
     make_large_pattern 131072 "$SCRATCH/large_want"
     cmp -s "$SCRATCH/large_want" "$SCRATCH/body" || \
         fail "128 KiB body differs from the expected pattern at $(cmp "$SCRATCH/large_want" "$SCRATCH/body" 2>&1 | head -1)"
     ok
 
     # POST /echo — round-trip body
     printf 'hello-payload-%s' "$label" >"$SCRATCH/post_body"
     test_method "[$label] POST /echo (body round-trip)" \
         POST /echo "$SCRATCH/post_body" 200 "hello-payload-$label"
 
     # The same in the request direction, and at a size that does not
     # fit in one buffer: 128 KiB of random bytes posted to /echo and
     # compared with what comes back.  POST /upload below checks only
     # the count the upstream reports, so without this nothing in the
     # suite would notice a request body that arrived complete but
     # corrupt.
     msg "[$label] POST /echo 128 KiB round-trips byte for byte"
     dd if=/dev/urandom of="$SCRATCH/echo_big" bs=1024 count=128 status=none
     local estatus
     estatus="$(curl -sS -X POST --data-binary "@$SCRATCH/echo_big" \
                     -o "$SCRATCH/body" -w '%{http_code}' \
                     "$(PAIVANA_URL /echo)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$estatus" = "200" ] || fail "status=$estatus want=200"
     cmp -s "$SCRATCH/echo_big" "$SCRATCH/body" || \
         fail "echoed 128 KiB body differs: $(cmp "$SCRATCH/echo_big" "$SCRATCH/body" 2>&1 | head -1)"
     ok
 
     # POST /upload — byte count
     dd if=/dev/urandom of="$SCRATCH/rnd" bs=1024 count=64 status=none
     test_method "[$label] POST /upload (64 KiB binary upload)" \
         POST /upload "$SCRATCH/rnd" 200 "Received 65536 bytes"
 
     # PUT /put
     test_method "[$label] PUT /put (PUT forwarding)" \
         PUT /put "$SCRATCH/post_body" 200 "PUT received"
 
     # PATCH /patch
     test_method "[$label] PATCH /patch (PATCH forwarding)" \
         PATCH /patch "$SCRATCH/post_body" 200 "PATCH received"
 
     # DELETE /item/1 with empty body
     msg "[$label] DELETE /item/1 (204 No Content)"
     local status
     status="$(curl -sS -X DELETE -o /dev/null -w '%{http_code}' "$(PAIVANA_URL /item/1)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "204" ] || fail "status=$status"
     ok
 
     # OPTIONS — server should echo 204 + Allow
     msg "[$label] OPTIONS /anything (204 + Allow header)"
     local opts
     opts="$(curl -sS -X OPTIONS -D "$SCRATCH/hdrs" -o /dev/null -w '%{http_code}' "$(PAIVANA_URL /hello)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$opts" = "204" ] || fail "status=$opts"
     grep -qi '^allow:' "$SCRATCH/hdrs" || fail "no Allow header returned"
     ok
 
     # Header propagation: X-Forwarded-For must be added by paivana.
     msg "[$label] GET /echo-headers (X-Forwarded-For added)"
     curl -sS -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     grep -qi '^x-forwarded-for:' "$SCRATCH/body" || \
         fail "upstream did not see X-Forwarded-For; headers:"$'\n'"$(cat "$SCRATCH/body")"
     grep -qi '^x-forwarded-proto:' "$SCRATCH/body" || \
         fail "upstream did not see X-Forwarded-Proto"
     grep -qi '^via:' "$SCRATCH/body" || \
         fail "upstream did not see Via: paivana"
     ok
 
     # RFC 9110 §7.2: the Host we send upstream names the *upstream*
     # authority, not the one the client dialed, and carries nothing
     # but host[:port] — no userinfo, no path, no query.  `build_host_header`
     # derives it from DESTINATION_BASE_URL.
     msg "[$label] Host header sent upstream names the upstream authority"
     local want_host stripped seen_host
     stripped="${PAIVANA_DEST#*://}"   # drop scheme
     want_host="${stripped%%/*}"       # drop any path
     seen_host="$(grep -i '^host:' "$SCRATCH/body" | tr -d '\r' | \
                  sed -e 's/^[Hh][Oo][Ss][Tt]: *//')"
     [ -n "$seen_host" ] || \
         fail "upstream saw no Host header; headers:"$'\n'"$(cat "$SCRATCH/body")"
     [ "$seen_host" = "$want_host" ] || \
         fail "upstream saw 'Host: $seen_host', want 'Host: $want_host'"
     ok
 
     # RFC 9110 §7.6.3: client's Via chain must be preserved and our
     # pseudonym *appended* to it, not replaced.
     msg "[$label] client Via is preserved and paivana is appended"
     curl -sS -H 'Via: 1.1 alpha.example, 2.0 beta.example' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local via
     via="$(grep -i '^via:' "$SCRATCH/body" | tr -d '\r')"
     [ -n "$via" ] || fail "no Via header at upstream"
     # Expect: "Via: 1.1 alpha.example, 2.0 beta.example, 1.1 paivana"
     case "$via" in
         *"alpha.example"*"beta.example"*paivana*) ;;
         *) fail "Via not appended correctly: '$via'";;
     esac
     ok
 
     # RFC 9110 §7.6.1: headers named in the client's Connection
     # header are hop-by-hop and must not be forwarded upstream.
     msg "[$label] headers named in Connection: are stripped"
     curl -sS \
          -H 'Connection: X-Custom-Hop, X-Other-Hop' \
          -H 'X-Custom-Hop: must-not-forward' \
          -H 'X-Other-Hop: neither' \
          -H 'X-Keep: keep-this' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     if grep -qi '^x-custom-hop:' "$SCRATCH/body";
     then
         fail "X-Custom-Hop leaked to upstream (Connection list ignored)"
     fi
     if grep -qi '^x-other-hop:' "$SCRATCH/body";
     then
         fail "X-Other-Hop leaked to upstream (Connection list ignored)"
     fi
     grep -qi '^x-keep:.*keep-this' "$SCRATCH/body" || \
         fail "X-Keep (not named in Connection) was incorrectly dropped"
     ok
 
     # Custom request header must be forwarded.
     msg "[$label] custom request header X-Test is forwarded"
     curl -sS -H 'X-Test: dingbat-42' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     grep -qi '^x-test:.*dingbat-42' "$SCRATCH/body" || \
         fail "upstream did not see X-Test: dingbat-42"
     ok
 
     # Our access cookie is a credential for paivana itself; the
     # origin must never see it, while the cookies that are genuinely
     # the origin's have to survive verbatim.  The name match is
     # case-insensitive (that is how MHD looks it up) but exact: names
     # that merely contain it are somebody else's cookies.
     msg "[$label] Paivana-Cookie is stripped from the forwarded Cookie"
     curl -sS \
          -H 'Cookie: sid=alpha;paivana-cookie=1234-secret; theme=dark' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     if grep -qi '^cookie:.*paivana-cookie' "$SCRATCH/body";
     then
         fail "access cookie leaked upstream: $(grep -i '^cookie:' "$SCRATCH/body")"
     fi
     grep -qi '^cookie:.*sid=alpha' "$SCRATCH/body" || \
         fail "client cookie sid=alpha was dropped"
     grep -qi '^cookie:.*theme=dark' "$SCRATCH/body" || \
         fail "client cookie theme=dark was dropped"
     ok
 
     # If the access cookie was the only one, no Cookie header at all
     # should reach the origin -- not an empty one.
     msg "[$label] lone Paivana-Cookie leaves no Cookie header"
     curl -sS -H 'Cookie: Paivana-Cookie=1234-secret' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     if grep -qi '^cookie:' "$SCRATCH/body";
     then
         fail "unexpected Cookie header upstream: $(grep -i '^cookie:' "$SCRATCH/body")"
     fi
     ok
 
     # Cookies whose name merely embeds ours are not ours.
     msg "[$label] cookies named like ours are not over-stripped"
     curl -sS \
          -H 'Cookie: Paivana-Cookie-2=keep; XPaivana-Cookie=keep2' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     grep -qi '^cookie:.*paivana-cookie-2=keep' "$SCRATCH/body" || \
         fail "Paivana-Cookie-2 was incorrectly stripped"
     grep -qi '^cookie:.*xpaivana-cookie=keep2' "$SCRATCH/body" || \
         fail "XPaivana-Cookie was incorrectly stripped"
     ok
 
     # RFC 9110 §7.6.1, response direction: headers named in the
     # *upstream's* Connection header are equally hop-by-hop and must
     # not be relayed to the client.  The upstream emits one such
     # header before the Connection line and one after it, so this
     # also pins that the filter is applied to the complete header
     # block rather than as the headers stream in.
     msg "[$label] headers named in upstream Connection: are stripped"
     curl -sS -D "$SCRATCH/hdrs" -o /dev/null \
          "$(PAIVANA_URL /conn-response)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     if grep -qi '^x-hop-before:' "$SCRATCH/hdrs";
     then
         fail "X-Hop-Before leaked to client (upstream Connection ignored)"
     fi
     if grep -qi '^x-hop-after:' "$SCRATCH/hdrs";
     then
         fail "X-Hop-After leaked to client (upstream Connection ignored)"
     fi
     grep -qi '^x-keep-resp:.*survivor' "$SCRATCH/hdrs" || \
         fail "X-Keep-Resp (not named in Connection) was incorrectly dropped"
     ok
 
     # Response header passthrough: upstream sets X-Upstream.  Its
     # *value* is checked against the label, not merely its presence:
     # the point of the header is to confirm which of the four servers
     # answered, and every case in this battery is run against a
     # paivana we have just re-pointed.  A restart that silently kept
     # the previous destination would pass a presence check.
     msg "[$label] upstream response header X-Upstream is forwarded"
     curl -sS -D "$SCRATCH/hdrs" -o /dev/null "$(PAIVANA_URL /hello)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local seen_upstream
     seen_upstream="$(grep -i '^x-upstream:' "$SCRATCH/hdrs" | tr -d '\r' | \
                      sed -e 's/^[^:]*: *//')"
     [ -n "$seen_upstream" ] || \
         fail "X-Upstream header not forwarded back to client"
     [ "$seen_upstream" = "$label" ] || \
         fail "X-Upstream='$seen_upstream', want '$label' (answered by the wrong upstream)"
     ok
 }
 
 ######################################################################
 # Cross-cutting tests (do not depend on which upstream is used).
 ######################################################################
 
 function test_method_not_allowed() {
     msg "unsupported HTTP method (TRACE) yields 405"
     local status
     status="$(curl -sS -X TRACE -o /dev/null -w '%{http_code}' \
                    "$(PAIVANA_URL /hello)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "405" ] || fail "status=$status want=405"
     ok
 }
 
 function test_upload_too_big() {
     msg "upload exceeding 1 MiB buffer yields 413"
     dd if=/dev/zero of="$SCRATCH/big" bs=1024 count=2048 status=none
     local status
     status="$(curl -sS -X POST --data-binary "@$SCRATCH/big" \
                    -o /dev/null -w '%{http_code}' \
                    "$(PAIVANA_URL /upload)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     # 413 == Content Too Large; some builds report 500 on hook close
     [ "$status" = "413" ] || fail "status=$status want=413"
     ok
 }
 
 function test_upload_too_big_early() {
     # Open a raw TCP connection and send a POST whose Content-Length
     # already exceeds the buffer cap, but DON'T send any body bytes.
     # Paivana must reject on the Content-Length header alone (during
     # MHD's HEADERS_PROCESSED callback), respond with 413 and close
     # the connection.  If the early-reject path is missing the server
     # would block waiting for a body that never arrives and we'd hit
     # the timeout, which fails the test rather than masquerading as
     # a pass.
     msg "Content-Length exceeding 1 MiB triggers early 413 (no body sent)"
     local out
     out="$( ( exec 3<>"/dev/tcp/127.0.0.1/$PAIVANA_PORT"
               printf 'POST /upload HTTP/1.1\r\nHost: 127.0.0.1:%s\r\nContent-Length: 10485760\r\nConnection: close\r\n\r\n' \
                      "$PAIVANA_PORT" >&3
               timeout 5 cat <&3 ) 2>"$SCRATCH/err")" \
         || fail "raw POST failed (timeout or socket error): $(cat "$SCRATCH/err")"
     case "$out" in
         'HTTP/1.1 413'*) ;;
         *) fail "expected 413 status line, got: $(echo "$out" | head -c 80)";;
     esac
     ok
 }
 
 function test_upload_too_big_no_continue() {
     # When the client opts in to 100-continue, paivana must NOT send
     # the interim 100 response if it has already decided to reject
     # the upload — it should jump straight to 413.  Use curl with
     # `Expect: 100-continue` and verbose tracing, then assert that
     # no `< HTTP/1.1 100` line appeared on the wire.
     msg "rejection suppresses 100 Continue when client opts in"
     dd if=/dev/zero of="$SCRATCH/big" bs=1024 count=2048 status=none
     local status
     status="$(curl -sSv -X POST -H 'Expect: 100-continue' \
                    --expect100-timeout 5 \
                    --data-binary "@$SCRATCH/big" \
                    -o /dev/null -w '%{http_code}' \
                    "$(PAIVANA_URL /upload)" 2>"$SCRATCH/trace")" \
         || fail "curl: $(cat "$SCRATCH/trace")"
     [ "$status" = "413" ] || fail "status=$status want=413"
     if grep -q '^< HTTP/1.1 100' "$SCRATCH/trace";
     then
         fail "server sent 100 Continue before 413; trace: $(grep '^<' "$SCRATCH/trace")"
     fi
     ok
 }
 
 function test_upload_too_big_chunked() {
     # Chunked transfer-encoding has no Content-Length, so paivana
     # cannot know the upload is too big until it actually reaches
     # the cap mid-stream.  This test guards the fallback
     # drain-then-reject path that runs in BODY_RECEIVING /
     # FULL_REQ_RECEIVED.
     msg "chunked upload exceeding 1 MiB still yields 413 (drain path)"
     dd if=/dev/zero of="$SCRATCH/big" bs=1024 count=2048 status=none
     local status
     status="$(curl -sS -X POST \
                    -H 'Transfer-Encoding: chunked' \
                    -H 'Content-Length:' \
                    --data-binary "@$SCRATCH/big" \
                    -o /dev/null -w '%{http_code}' \
                    "$(PAIVANA_URL /upload)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "413" ] || fail "status=$status want=413"
     ok
 }
 
 # 768 KiB — under paivana's 1 MiB request-buffer cap, but far too
 # large to fit in kernel TCP buffers, so the upstream is guaranteed
 # to have answered while paivana's curl is still uploading.
 EARLY_PAYLOAD_SIZE=786432
 
 # Write the early-response payload to $SCRATCH/early_body, once.
 function make_early_payload() {
     local payload="$SCRATCH/early_body"
     local got
     [ -s "$payload" ] && return 0
     dd if=/dev/urandom of="$payload" bs=1024 count=768 status=none
     got="$(wc -c <"$payload" | tr -d ' ')"
     [ "$got" = "$EARLY_PAYLOAD_SIZE" ] || \
         fail "test setup: payload is $got bytes, want $EARLY_PAYLOAD_SIZE"
 }
 
 EARLY_PID=""
 
 # Start early_response_upstream.  $1 = port, $2 = receipt file, and
 # any further arguments go to the upstream (e.g. --no-drain).
 function start_early_upstream() {
     local port="$1" receipt="$2"; shift 2
     rm -f "$receipt"
     ( exec "$EARLY_RESPONSE_UPSTREAM" "$port" "$receipt" "$@" ) \
         >"$LOGDIR/early-$port.log" 2>&1 &
     EARLY_PID=$!
     PIDS+=("$EARLY_PID")
     if ! wait_for_port 127.0.0.1 "$port" "$EARLY_PID";
     then
         fail "early_response_upstream did not start on port $port"
     fi
 }
 
 function stop_early_upstream() {
     if [ -n "$EARLY_PID" ];
     then
         kill -TERM "$EARLY_PID" 2>/dev/null
         # In --no-drain mode the upstream may be parked on a
         # connection whose peer never hangs up, so back the TERM with
         # a KILL after two seconds: a wedged helper must not wedge the
         # whole suite.
         #
         # Polled here rather than armed as `( sleep 2; kill -KILL ) &'
         # and cancelled afterwards.  That subshell inherits this
         # script's EXIT trap, and the `kill -TERM' that cancels it
         # makes bash run that trap *in the subshell*: cleanup() then
         # rm -rf's the scratch directory and TERMs every helper while
         # the suite is still running, and the run dies several checks
         # later reading "logs/early-<port>.log: No such file or
         # directory".  Reproduced in isolation, and seen once in
         # roughly twenty runs of the suite -- only when the wait
         # really does take the full two seconds, which is the case
         # the watchdog exists for.
         local tries=20
         while [ "$tries" -gt 0 ] && kill -0 "$EARLY_PID" 2>/dev/null;
         do
             sleep 0.1
             tries=$((tries - 1))
         done
         kill -KILL "$EARLY_PID" 2>/dev/null
         wait "$EARLY_PID" 2>/dev/null
         EARLY_PID=""
     fi
 }
 
 # Block until the upstream's receipt file appears (it writes it once
 # the connection is over), then echo the count it recorded.  Echoes
 # nothing if it never showed up.
 function read_receipt() {
     local receipt="$1" tries=50
     while [ "$tries" -gt 0 ] && [ ! -s "$receipt" ];
     do
         sleep 0.1
         tries=$((tries - 1))
     done
     [ -s "$receipt" ] || return 0
     tr -d '\n ' <"$receipt"
 }
 
 function test_early_response() {
     # The upstream sends a 413 response immediately after reading the
     # request headers, before consuming any of the request body — a
     # legal HTTP/1.1 pattern.  Paivana must deliver that response back
     # to its own client rather than turning it into a 502, even though
     # its curl handle was still uploading when it arrived.
     #
     # Note what is deliberately NOT asserted: that the upstream sees
     # the whole body.  RFC 9110 §9.3 lets a client stop sending once
     # it has a final response, and libcurl does exactly that — plain
     # `curl` against this upstream sends ~128 KiB of a 768 KiB body
     # and stops.  The receipt the upstream writes is therefore a
     # diagnostic, and all we require of it is that it appears: that
     # means the exchange finished upstream-side instead of leaving
     # the upstream blocked in read() forever.
     msg "early upstream response is forwarded to the client"
     if [ ! -x "$EARLY_RESPONSE_UPSTREAM" ];
     then
         echo "SKIP (early_response_upstream not built)"
         return
     fi
     stop_paivana
     local receipt="$SCRATCH/early_receipt"
     start_early_upstream "$EARLY_PORT" "$receipt"
     start_paivana "http://127.0.0.1:$EARLY_PORT"
     make_early_payload
 
     # Without the early-response fix paivana would abort the curl
     # handle and answer 502 instead of relaying what it was handed.
     local status
     status="$(curl -sS -X POST --data-binary "@$SCRATCH/early_body" \
                    --max-time 30 \
                    -o "$SCRATCH/body" -w '%{http_code}' \
                    "$(PAIVANA_URL /upload)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "413" ] || \
         fail "status=$status want=413 (a 502 here means paivana aborted the forward instead of relaying the early response)"
     grep -q 'early-response-payload' "$SCRATCH/body" || \
         fail "response body did not come from upstream: $(head -c 200 "$SCRATCH/body")"
 
     [ -n "$(read_receipt "$receipt")" ] || \
         fail "upstream never finished the exchange (still blocked reading the body?)"
     ok
 
     stop_paivana
     stop_early_upstream
 }
 
 function test_early_response_no_drain() {
     # The dangerous half of the same situation.  Here the upstream
     # answers early and then refuses to read another byte, leaving
     # the rest of the request queued in its (deliberately tiny)
     # receive buffer.  Paivana's outbound socket therefore stays full
     # and its write() can never complete: the only way out is to act
     # on the response it already holds.  A proxy that instead insists
     # on finishing the upload first deadlocks until its own transfer
     # timeout (60s for us) and only then answers 502 — which is why
     # `timeout` bounds this case.  Without that bound the hang would
     # eventually "pass" as a slow 502 rather than fail.
     #
     # The drain-mode test above cannot catch this: because that
     # upstream keeps read()-ing until EOF, paivana's socket never
     # stays full and the deadlock never has a chance to form.
     msg "early upstream response, upstream then stops reading: no hang"
     if [ ! -x "$EARLY_RESPONSE_UPSTREAM" ];
     then
         echo "SKIP (early_response_upstream not built)"
         return
     fi
     stop_paivana
     local receipt="$SCRATCH/nodrain_receipt"
     start_early_upstream "$NODRAIN_PORT" "$receipt" --no-drain
     start_paivana "http://127.0.0.1:$NODRAIN_PORT"
     make_early_payload
 
     # 20s is comfortably above a healthy round-trip (milliseconds)
     # and comfortably below paivana's 60s CURLOPT_TIMEOUT, so a
     # deadlock shows up as a killed curl rather than a late answer.
     local status rc
     status="$(timeout 20 curl -sS -X POST \
                       --data-binary "@$SCRATCH/early_body" \
                       -o "$SCRATCH/body" -w '%{http_code}' \
                       "$(PAIVANA_URL /upload)" 2>"$SCRATCH/err")"
     rc=$?
     [ "$rc" != "124" ] && [ "$rc" != "137" ] || \
         fail "no response within 20s: paivana blocked on an upload the upstream stopped reading"
     [ "$rc" = "0" ] || fail "curl exited $rc: $(cat "$SCRATCH/err")"
     [ "$status" = "413" ] || \
         fail "status=$status want=413 (a 502 means paivana gave up on the transfer instead of using the response it had)"
     grep -q 'early-response-payload' "$SCRATCH/body" || \
         fail "response body did not come from upstream: $(head -c 200 "$SCRATCH/body")"
     # No receipt assertion here: the upstream only ever saw whatever
     # fit in its receive buffer, and whether the connection is closed
     # or kept for reuse afterwards is paivana's business.  The count
     # it does eventually record is for diagnostics.
     ok
 
     stop_paivana
     stop_early_upstream
 }
 
 
 function start_truncating_upstream() {
     # An upstream that declares more body than it delivers and then
     # closes: the wire shape of a response that arrived incomplete.
     # It is also exactly what paivana ends up holding when
     # libgnunetcurl stops buffering at its 40 MiB ceiling -- libcurl
     # fails the transfer, but the status line it parsed long before is
     # still what CURLINFO_RESPONSE_CODE reports.
     local port="$1"
     cat >"$SCRATCH/truncating_upstream.py" <<'PYEOF'
 import socket
 import sys
 
 port = int(sys.argv[1])
 srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
 srv.bind(("127.0.0.1", port))
 srv.listen(5)
 while True:
     conn, _ = srv.accept()
     try:
         conn.recv(65536)
         conn.sendall(b"HTTP/1.1 200 OK\r\n"
                      b"Content-Type: text/plain\r\n"
                      b"Content-Length: 100000\r\n"
                      b"\r\n"
                      + b"x" * 1000)
     except OSError:
         pass
     conn.close()
 PYEOF
     local log="$LOGDIR/truncating.log"
     ( exec python3 "$SCRATCH/truncating_upstream.py" "$port" ) >"$log" 2>&1 &
     local tpid=$!
     PIDS+=("$tpid")
     if ! wait_for_port 127.0.0.1 "$port" "$tpid";
     then
         echo "FAIL: truncating upstream did not start on port $port" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 }
 
 # ======================================================================
 # Streaming
 #
 # The point of the whole streaming path is that a proxied body is no
 # longer bounded by memory, and that it starts reaching the client
 # before the origin has finished sending.  Neither is visible in the
 # battery above, where every body fits in one buffer and would have
 # done so before.
 #
 # `stream_upstream' serves bodies generated from their own offset, and
 # `stream_client' verifies them the same way as they arrive, so a
 # 200 MiB case costs no disk on either side and a duplicated or dropped
 # block is caught rather than just a wrong total.  The client prints
 # `key=value' lines; `sfield' pulls one out.
 # ======================================================================
 
 # Bytes used for the "large" cases.  200 MiB is five times the 40 MiB
 # ceiling that used to make these a 502, which is the point.
 #
 # PAIVANA_TEST_SCALE divides it, for the sanitised build where 200 MiB
 # is a coffee break rather than a test.  Dividing is sound here because
 # what these cases exercise is the pause/resume interleaving, and that
 # is a function of the ring size -- which does not scale -- rather than
 # of the total: a tenth of the bytes still crosses the ring hundreds of
 # times.  It is not sound for the *bound* being checked, so the 413
 # cases and the buffer sizes are left alone.
 STREAM_BIG=$(( (200 * 1024 * 1024) / ${PAIVANA_TEST_SCALE:-1} ))
 
 function start_stream_upstream() {
     local port="$1"
     local log="$LOGDIR/stream.log"
 
     ( exec "$BUILDDIR/stream_upstream" "$port" ) >"$log" 2>&1 &
     local spid=$!
     PIDS+=("$spid")
     if ! wait_for_port 127.0.0.1 "$port" "$spid";
     then
         echo "FAIL: stream upstream did not start on port $port" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 }
 
 # Run stream_client against the streaming upstream through paivana and
 # leave its report in $SCRATCH/sc.  Never fails the test itself: which
 # way a transfer went wrong is what the cases assert on.
 function sc() {
     local path="$1"; shift
 
     if ! timeout 300 "$BUILDDIR/stream_client" \
          "$(PAIVANA_URL "$path")" "$@" > "$SCRATCH/sc" 2>"$SCRATCH/scerr";
     then
         fail "stream_client did not finish for $path: $(cat "$SCRATCH/scerr")"
     fi
 }
 
 # Value of one `key=value' line of the last sc() report.
 function sfield() {
     sed -n "s/^$1=//p" "$SCRATCH/sc"
 }
 
 # Assert that a field of the last sc() report has the expected value.
 function sc_is() {
     local key="$1" want="$2" got
     got="$(sfield "$key")"
     [ "$got" = "$want" ] || \
         fail "$key=$got want=$want ($(tr '\n' ' ' < "$SCRATCH/sc"))"
 }
 
 function test_streaming() {
     stop_paivana
     start_stream_upstream "$STREAM_PORT"
     # A short stall timeout so the "origin goes quiet" cases cost
     # seconds rather than the minute the shipped default allows, and a
     # request cap above the large upload cases.
     local cfg="$SCRATCH/paivana-stream.conf"
     sed -e "s|@DEST@|http://127.0.0.1:$STREAM_PORT|g" \
         -e "s|@PORT@|$PAIVANA_PORT|g" \
         "$SRCDIR/test_reverse_proxy.conf.in" > "$cfg"
     cat >> "$cfg" <<EOF
 UPSTREAM_TIMEOUT = 3 s
 UPSTREAM_STALL_TIMEOUT = 3 s
 MAX_REQUEST_SIZE = $((512 * 1024 * 1024))
 EOF
     PAIVANA_DEST="http://127.0.0.1:$STREAM_PORT"
     local log="$LOGDIR/paivana.log"
     ( exec "$PAIVANA_HTTPD" -c "$cfg" -n -L WARNING ) >"$log" 2>&1 &
     PAIVANA_PID=$!
     if ! wait_for_port 127.0.0.1 "$PAIVANA_PORT" "$PAIVANA_PID";
     then
         echo "FAIL: paivana-httpd did not start on port $PAIVANA_PORT" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 
     # --- 1: a body five times the old ceiling, with a length --------
     msg "streams a 200 MiB response with Content-Length"
     sc "/cl?bytes=$STREAM_BIG" --expect-bytes "$STREAM_BIG"
     sc_is status 200
     sc_is curl 0
     sc_is pattern ok
     sc_is bytes "$STREAM_BIG"
     # The origin's own framing, not one recomputed from a buffer.
     sc_is content_length "$STREAM_BIG"
     sc_is chunked no
     ok
 
     # --- 2: the same body, chunked ----------------------------------
     msg "streams a 200 MiB chunked response, still chunked"
     sc "/chunked?bytes=$STREAM_BIG" --expect-bytes "$STREAM_BIG"
     sc_is status 200
     sc_is curl 0
     sc_is pattern ok
     sc_is bytes "$STREAM_BIG"
     # A chunked origin must not be silently converted to a declared
     # length, which is what buffering the body did.
     sc_is chunked yes
     sc_is content_length none
     ok
 
     # --- 3: chunked origin, HTTP/1.0 client -------------------------
     # An HTTP/1.0 client cannot be sent chunks, so the end of the body
     # has to be the close of the connection.
     msg "chunked upstream is close-delimited for an HTTP/1.0 client"
     local out
     out="$(curl -sS --http1.0 -o "$SCRATCH/h10" -D "$SCRATCH/h10hdr" \
                 -w '%{http_code}' --max-time 120 \
                 "$(PAIVANA_URL "/chunked?bytes=1048576")" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$out" = "200" ] || fail "status=$out want=200"
     grep -qi '^Transfer-Encoding:' "$SCRATCH/h10hdr" && \
         fail "chunked encoding offered to an HTTP/1.0 client"
     [ "$(wc -c < "$SCRATCH/h10")" = "1048576" ] || \
         fail "got $(wc -c < "$SCRATCH/h10") bytes, want 1048576"
     ok
 
     # --- 4: a range request through the stream --------------------
     msg "206 and Content-Range pass through a streamed response"
     out="$(curl -sS -r 1000-1999 -o "$SCRATCH/rng" -D "$SCRATCH/rnghdr" \
                 -w '%{http_code}' --max-time 60 \
                 "$(PAIVANA_URL "/range?bytes=1048576")" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$out" = "206" ] || fail "status=$out want=206"
     grep -qi '^Content-Range: bytes 1000-1999/1048576' "$SCRATCH/rnghdr" || \
         fail "no matching Content-Range: $(grep -i content-range "$SCRATCH/rnghdr")"
     [ "$(wc -c < "$SCRATCH/rng")" = "1000" ] || \
         fail "got $(wc -c < "$SCRATCH/rng") bytes, want 1000"
     ok
 
     # --- 5: HEAD on a large resource --------------------------------
     # MHD does not run the content reader for a HEAD but does emit the
     # size the response was created with, so the length the equivalent
     # GET would have had now reaches the client (RFC 9110 9.3.2).
     # Buffering could only ever have reported 0 here.
     msg "HEAD reports the upstream's length without a body"
     sc "/cl?bytes=$STREAM_BIG" --head
     sc_is status 200
     sc_is bytes 0
     sc_is content_length "$STREAM_BIG"
     ok
 
     # --- 6: statuses that carry no body -----------------------------
     msg "204 carries no body and no length"
     sc "/status?code=204" --print-body
     sc_is status 204
     sc_is bytes 0
     sc_is content_length none
     ok
 
     msg "304 keeps the length of the body it does not send"
     sc "/status?code=304&len=12345" --print-body
     sc_is status 304
     sc_is bytes 0
     sc_is content_length 12345
     ok
 
     # --- 7-8: the upload direction ----------------------------------
     # The origin reports what it received; `pattern=ok' in its report
     # is the byte-exactness assertion, and `framing=' is the assertion
     # that the client's own framing was reproduced upstream rather
     # than rewritten.
     msg "streams a 200 MiB request body with Content-Length"
     sc "/sink" --upload "$STREAM_BIG" --print-body
     sc_is status 200
     case "$(sfield body)" in
         "bytes=$STREAM_BIG framing=length pattern=ok") ;;
         *) fail "upstream saw: $(sfield body)" ;;
     esac
     ok
 
     msg "streams a 200 MiB chunked request body, still chunked"
     sc "/sink" --upload "$STREAM_BIG" --chunked-upload --print-body
     sc_is status 200
     case "$(sfield body)" in
         "bytes=$STREAM_BIG framing=chunked pattern=ok") ;;
         *) fail "upstream saw: $(sfield body)" ;;
     esac
     ok
 
     # --- 9: the common case, which now takes the same path ----------
     msg "a small POST still round-trips"
     sc "/sink" --upload 100 --print-body
     sc_is status 200
     case "$(sfield body)" in
         "bytes=100 framing=length pattern=ok") ;;
         *) fail "upstream saw: $(sfield body)" ;;
     esac
     ok
 
     # --- 18: chunked response with no terminating chunk -------------
     # The status is long gone by the time the origin gives up, so the
     # only remaining way to say "this is incomplete" is to close
     # without the terminator.  curl 18 is the client noticing.
     msg "a chunked upstream that stops mid-stream truncates the client"
     sc "/chunk-abort?after=5000"
     sc_is status 200
     sc_is chunked yes
     sc_is bytes 5000
     sc_is curl 18
     ok
 
     # --- 19: the stall watchdog -------------------------------------
     # An origin that sends headers and some body and then goes quiet
     # for ever.  MHD will not time this out -- a suspended connection
     # is off its timeout lists -- and CURLOPT_TIMEOUT is deliberately
     # unset, so paivana's own watchdog is the only thing that can end
     # it.  Without it the client hangs until it gives up itself.
     msg "an upstream that goes quiet is cut off by the stall watchdog"
     local t0 t1
     t0="$(date +%s)"
     sc "/hang?after=1000"
     t1="$(date +%s)"
     sc_is status 200
     sc_is bytes 1000
     sc_is curl 18
     [ "$((t1 - t0))" -lt 30 ] || \
         fail "took $((t1 - t0))s; the 3 s stall timeout did not fire"
     grep -q "moved no data" "$log" || \
         fail "no stall diagnostic in the log"
     ok
 
     # --- 20: an upstream that accepts and never answers -------------
     # Distinct from an upstream that is not there (502, tested
     # separately): this one is a 504, and the time-to-headers clock is
     # what tells them apart.  It is the only one of the three clocks
     # that can still produce a status code.
     msg "an upstream that never answers yields 504"
     out="$(curl -sS -o "$SCRATCH/body" -w '%{http_code}' --max-time 60 \
                 "$(PAIVANA_URL /mute)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$out" = "504" ] || fail "status=$out want=504"
     ok
 
     # --- 21: the client walks away mid-download ---------------------
     # The interesting part is not the one request but that a hundred of
     # them leave nothing behind: the response is queued and its content
     # reader is live for every one of these, so a mistake in the
     # ownership handshake between MHD's completion notifier and the
     # reader's free callback leaks (or worse) on each.
     msg "100 downloads abandoned mid-body leave no growth behind"
     local rss0 rss1
     sc "/cl?bytes=$STREAM_BIG" --abort-after 1048576
     rss0="$(awk '/VmRSS/{print $2}' "/proc/$PAIVANA_PID/status")"
     local i=0
     while [ "$i" -lt 100 ];
     do
         timeout 60 "$BUILDDIR/stream_client" \
                 "$(PAIVANA_URL "/cl?bytes=$STREAM_BIG")" \
                 --abort-after 1048576 >/dev/null 2>&1 || \
             fail "stream_client did not finish on iteration $i"
         i=$((i + 1))
     done
     rss1="$(awk '/VmRSS/{print $2}' "/proc/$PAIVANA_PID/status")"
     if [ -n "${PAIVANA_SANITIZED:-}" ];
     then
         # RSS is not a leak detector under ASan: redzones around every
         # allocation and a quarantine that deliberately withholds freed
         # memory make the process grow whether or not anything leaked.
         # The hundred iterations above still ran, and LSan is watching
         # them -- which is a far better detector than this bound.  It
         # is this bound that is the stand-in, for the build where LSan
         # is not there.
         echo "OK (RSS bound not meaningful under sanitizers; LSan covers it)"
     else
         # A generous bound: the point is "flat", not "identical".  Real
         # per-request leakage of a ring or a response would be megabytes
         # over a hundred iterations.
         [ "$((rss1 - rss0))" -lt 4096 ] || \
             fail "RSS grew ${rss0}k -> ${rss1}k over 100 abandoned downloads"
         ok
     fi
 
     # --- 22: the client walks away mid-upload -----------------------
     # We have declared a Content-Length upstream that we can no longer
     # deliver, so the origin has to be told the request is broken
     # rather than left waiting for bytes that will never come.
     msg "an upload abandoned by the client does not wedge paivana"
     timeout 60 curl -sS -o /dev/null --max-time 2 \
             --data-binary "@$SCRATCH/echo_big" \
             "$(PAIVANA_URL "/sink?rate=20000")" >/dev/null 2>&1
     # Whatever that did to the one request, the daemon must still be
     # serving; a wedged read callback would take the event loop with
     # it.
     sc "/cl?bytes=1024" --expect-bytes 1024
     sc_is status 200
     sc_is pattern ok
     ok
 
     # --- 23: the origin answers during a large upload ---------------
     # Only reachable because the request body is streamed: with it
     # buffered first, the origin could not have answered before seeing
     # all of it.  The client must get the origin's 413, not a 502, and
     # the drain must complete rather than deadlock.
     msg "an early 413 during a 200 MiB upload reaches the client"
     sc "/sink-early?after=1048576" --upload "$STREAM_BIG" --print-body
     sc_is status 413
     ok
 
     # --- 24-25: what 0034 established, on the streamed path ---------
     msg "trailers on a streamed chunked response are still dropped"
     out="$(curl -sS -o "$SCRATCH/tr" -D "$SCRATCH/trhdr" \
                 -w '%{http_code}' --max-time 60 \
                 "$(PAIVANA_URL "/trailers?bytes=4096")" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$out" = "200" ] || fail "status=$out want=200"
     grep -qi 'X-Trailer-Check' "$SCRATCH/trhdr" && \
         fail "a trailer field was merged into the header section"
     ok
 
     msg "a 1xx before a streamed response is not merged into it"
     out="$(curl -sS -o "$SCRATCH/ih" -D "$SCRATCH/ihhdr" \
                 -w '%{http_code}' --max-time 60 \
                 "$(PAIVANA_URL "/interim?bytes=4096")" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$out" = "200" ] || fail "status=$out want=200"
     grep -qi 'X-Interim-Check' "$SCRATCH/ihhdr" && \
         fail "an interim-response header reappeared on the final response"
     [ "$(wc -c < "$SCRATCH/ih")" = "4096" ] || \
         fail "got $(wc -c < "$SCRATCH/ih") bytes, want 4096"
     ok
 
     stop_paivana
 }
 
 
 # ======================================================================
 # Congestion, and the bound being real
 #
 # The cases in test_streaming show that a large body gets through
 # intact.  They do not show that it got through *without being held in
 # memory*, and they would all pass just as well against a version that
 # quietly buffered the lot -- so on their own the central claim of the
 # whole change is untested.  These are the cases that test it.
 #
 # Three things are measured that the client cannot see on its own:
 #
 #   - paivana's VmRSS while a large body is in flight, which is the
 #     bound itself;
 #   - how long the *origin* took to write its body, which is the
 #     backpressure.  A proxy that buffers takes everything at line rate
 #     however slowly its client reads; one that relays can only take
 #     what the client has made room for.  From the client end the two
 #     look identical, which is why the origin reports its own timing;
 #   - paivana's CPU time across an interval when nothing is moving,
 #     which is the busy-wait detector.  Spinning is the classic failure
 #     of a suspend/resume design and is otherwise invisible: the
 #     transfer still completes, just with a core pinned.
 #
 # Rate limits are what make any of this reproducible.  On loopback with
 # both ends going flat out, the kernel socket buffers absorb everything
 # and no ring ever fills.
 # ======================================================================
 
 # Sizes for these cases, in bytes.  Smaller than test_streaming's,
 # because each is deliberately slowed to a few seconds and the point
 # here is the shape of the flow rather than the total.
 #
 # Deliberately NOT divided by PAIVANA_TEST_SCALE, unlike test_streaming.
 # Every one of these is rate-limited, so its duration is set by the rate
 # and not by the size, and the sanitised build is no slower for them.
 # Scaling them down would also break the pacing assertions outright: the
 # kernel socket buffers hold a fixed couple of megabytes however small
 # the body is, so at a twentieth of the size the origin legitimately
 # finishes well ahead of the client and "was it throttled" stops having
 # a stable answer.
 CONG_BIG=$((64 * 1024 * 1024))
 CONG_MID=$((32 * 1024 * 1024))
 CONG_SMALL=$((8 * 1024 * 1024))
 
 # Resident set of a process, in kB.
 function rss_kb() {
     awk '/VmRSS/{print $2}' "/proc/$1/status" 2>/dev/null || echo 0
 }
 
 # User+system CPU of a process, in jiffies (100 per second).
 function cpu_jiffies() {
     awk '{print $14 + $15}' "/proc/$1/stat" 2>/dev/null || echo 0
 }
 
 # Sample rss_kb of $1 every 200 ms until none of the pids in $2.. are
 # left, leaving the maximum in $PEAK_RSS and the number of samples
 # taken in $RSS_SAMPLES.
 #
 # Deliberately given the pids to wait for rather than asking `jobs':
 # paivana itself is a background job of this same shell, so "wait while
 # any job is running" never becomes false and the sampler spins for
 # ever.  For the same reason callers must `wait' on the client pids by
 # name and not bare.
 #
 # The sample count is not bookkeeping.  If the transfer finishes before
 # the first tick, the loop never runs, the peak is whatever RSS was
 # before it started, and "no growth" is asserted about a measurement
 # that was never taken -- a silent vacuous pass on the one claim the
 # whole change rests on.  Callers must check it.
 function watch_rss() {
     local pid="$1"; shift
     local r p alive
     PEAK_RSS="$(rss_kb "$pid")"
     RSS_SAMPLES=0
     while true;
     do
         alive=0
         for p in "$@";
         do
             if kill -0 "$p" 2>/dev/null;
             then
                 alive=1
                 break
             fi
         done
         [ "$alive" = "0" ] && break
         r="$(rss_kb "$pid")"
         [ "${r:-0}" -gt "$PEAK_RSS" ] && PEAK_RSS="$r"
         RSS_SAMPLES=$((RSS_SAMPLES + 1))
         sleep 0.2
     done
 }
 
 # Assert $PEAK_RSS is no more than $2 kB above the baseline $1, and
 # report the measurement; $3 describes the transfer for the failure.
 #
 # Skipped under sanitizers, where RSS stops meaning what this case
 # needs it to mean.  ASan surrounds every allocation with redzones and,
 # more to the point, holds freed chunks in a quarantine rather than
 # reusing them -- that quarantine is exactly what lets it catch a
 # use-after-free, so turning it down to make this number readable would
 # trade away the thing the sanitised build exists for.  Measured: the
 # 64 MiB case grows ~58 MB instrumented against ~0.5 MB not, for
 # identical code.  LSan is the detector in that build; this is its
 # stand-in in the ordinary one.
 #
 # Printing the number rather than just "OK" is deliberate: a bound that
 # is never approached and a bound that was never measured look the same
 # from a pass, and this is the assertion the whole change rests on.
 function rss_bound() {
     local base="$1" limit="$2" what="$3"
 
     if [ -n "${PAIVANA_SANITIZED:-}" ];
     then
         echo "OK (RSS bound not meaningful under sanitizers;" \
              "saw +$((PEAK_RSS - base))k, LSan covers the leak side)"
         return
     fi
     [ "$((PEAK_RSS - base))" -lt "$limit" ] || \
         fail "RSS grew ${base}k -> ${PEAK_RSS}k $what"
     echo "OK (peak +$((PEAK_RSS - base))k over $RSS_SAMPLES samples)"
 }
 
 # ms the origin reported for the most recent request whose target
 # matches $1.  See the "served" line stream_upstream writes per
 # connection.
 function origin_ms() {
     tr -d '\0' < "$LOGDIR/stream.log" \
         | grep -a "^served target=$1 " | tail -1 \
         | sed -n 's/.* ms=//p'
 }
 
 function origin_bytes() {
     tr -d '\0' < "$LOGDIR/stream.log" \
         | grep -a "^served target=$1 " | tail -1 \
         | sed -n 's/.* bytes=\([0-9]*\) .*/\1/p'
 }
 
 function test_congestion() {
     stop_paivana
     # Points at the upstream test_streaming already started.  Its own
     # paivana, so the buffer sizes these cases assert against are
     # stated here rather than inherited.
     local cfg="$SCRATCH/paivana-congestion.conf"
     sed -e "s|@DEST@|http://127.0.0.1:$STREAM_PORT|g" \
         -e "s|@PORT@|$PAIVANA_PORT|g" \
         "$SRCDIR/test_reverse_proxy.conf.in" > "$cfg"
     cat >> "$cfg" <<EOF
 REQUEST_BUFFER_MAX = 262144
 RESPONSE_BUFFER_MAX = 262144
 MAX_REQUEST_SIZE = $((512 * 1024 * 1024))
 UPSTREAM_STALL_TIMEOUT = 30 s
 # The concurrency case runs 32 transfers from 127.0.0.1, which is
 # exactly PER_IP_CONNECTION_LIMIT's default: leaving it would have the
 # case measure connection limiting rather than the memory bound, and
 # would do so by refusing whichever request happened to be 33rd.
 PER_IP_CONNECTION_LIMIT = 0
 EOF
     PAIVANA_DEST="http://127.0.0.1:$STREAM_PORT"
     local log="$LOGDIR/paivana.log"
     ( exec "$PAIVANA_HTTPD" -c "$cfg" -n -L WARNING ) >"$log" 2>&1 &
     PAIVANA_PID=$!
     if ! wait_for_port 127.0.0.1 "$PAIVANA_PORT" "$PAIVANA_PID";
     then
         echo "FAIL: paivana-httpd did not start on port $PAIVANA_PORT" >&2
         tail -n 20 "$log" >&2
         exit 1
     fi
 
     # --- 10: the bound itself ---------------------------------------
     # A body many times the size of the buffers, through a client slow
     # enough that paivana cannot simply hand it straight on.  Peak RSS
     # over baseline is the assertion, and it is the one that makes the
     # rest of the suite mean anything: everything else here would pass
     # against a version that buffered the whole body.
     msg "a $((CONG_BIG / 1024 / 1024)) MiB download through a slow client stays within its buffers"
     local base peak
     base="$(rss_kb "$PAIVANA_PID")"
     "$BUILDDIR/stream_client" \
         "$(PAIVANA_URL "/cl?bytes=$CONG_BIG")" \
         --expect-bytes "$CONG_BIG" \
         --read-rate $((16 * 1024 * 1024)) > "$SCRATCH/sc" 2>&1 &
     local cpid=$!
     watch_rss "$PAIVANA_PID" "$cpid"
     wait "$cpid" || fail "stream_client did not finish"
     sc_is status 200
     sc_is pattern ok
     sc_is bytes "$CONG_BIG"
     # Generous: two 256 KiB rings, MHD's block buffer, libcurl's own
     # buffering and glibc's allocator.  The number that matters is that
     # it does not scale with the body -- buffering would show tens of
     # megabytes here, and CONG_BIG is far above the ceiling that used to
     # apply at all.
     [ "$RSS_SAMPLES" -ge 3 ] || \
         fail "only $RSS_SAMPLES RSS samples taken; the transfer was too fast to have measured anything"
     rss_bound "$base" 16384 "relaying $CONG_BIG bytes"
 
     # --- 11: the origin really was held back ------------------------
     # Same transfer, seen from the other end.  Without backpressure the
     # origin writes its body at loopback speed -- well under a second
     # for this size -- and paivana holds the difference.  With it, the
     # origin can only get as far ahead as the buffers allow, so its own
     # elapsed time tracks the client's.
     msg "the upstream is paced by the client rather than by the socket"
     local oms cms
     oms="$(origin_ms "/cl?bytes=$CONG_BIG")"
     cms="$(sfield total_ms)"
     [ -n "$oms" ] || fail "upstream reported no timing for /cl?bytes=$CONG_BIG"
     [ "$(origin_bytes "/cl?bytes=$CONG_BIG")" = "$CONG_BIG" ] || \
         fail "upstream wrote $(origin_bytes "/cl?bytes=$CONG_BIG") of $CONG_BIG bytes"
     # Half is the slack for the kernel socket buffers on both sides plus
     # the rings; the failure this is looking for is the origin finishing
     # in a fiftieth of the time, not in nine tenths of it.
     [ "$oms" -ge "$((cms / 2))" ] || \
         fail "upstream finished writing in ${oms}ms while the client took ${cms}ms: it was not throttled"
     echo "OK (upstream ${oms}ms, client ${cms}ms)"
 
     # --- 12: the same, in the upload direction ----------------------
     msg "a slow client uploading is likewise paced end to end"
     "$BUILDDIR/stream_client" "$(PAIVANA_URL /sink)" \
         --upload "$CONG_MID" --upload-rate $((16 * 1024 * 1024)) \
         --print-body > "$SCRATCH/sc" 2>&1 \
         || fail "stream_client did not finish"
     sc_is status 200
     case "$(sfield body)" in
         "bytes=$CONG_MID framing=length pattern=ok") ;;
         *) fail "upstream saw: $(sfield body)" ;;
     esac
     oms="$(origin_ms "/sink")"
     cms="$(sfield total_ms)"
     [ -n "$oms" ] || fail "upstream reported no timing for /sink"
     [ "$oms" -ge "$((cms / 2))" ] || \
         fail "upstream finished reading in ${oms}ms while the client took ${cms}ms"
     echo "OK (upstream ${oms}ms, client ${cms}ms)"
 
     # --- 13: many at once -------------------------------------------
     # The per-request cost is what multiplies, so this is where a bound
     # that holds for one request and not for sixteen would show.  Mixed
     # rates so the fast ones finish while the slow ones are still going,
     # which is the state a single-rate run never reaches.
     msg "32 concurrent throttled downloads stay within a bounded total"
     base="$(rss_kb "$PAIVANA_PID")"
     rm -f "$SCRATCH"/cong.*.out
     local i
     local cpids=""
     for i in $(seq 1 32);
     do
         "$BUILDDIR/stream_client" \
             "$(PAIVANA_URL "/cl?bytes=$CONG_SMALL")" \
             --expect-bytes "$CONG_SMALL" \
             --read-rate $(( (i % 4 + 1) * 4 * 1024 * 1024 )) \
             > "$SCRATCH/cong.$i.out" 2>&1 &
         cpids="$cpids $!"
     done
     # shellcheck disable=SC2086
     watch_rss "$PAIVANA_PID" $cpids
     # shellcheck disable=SC2086
     wait $cpids
     local okcount
     okcount="$(cat "$SCRATCH"/cong.*.out | grep -c '^pattern=ok$')"
     [ "$okcount" = "32" ] || \
         fail "only $okcount of 32 concurrent bodies verified"
     grep -hq '^status=200$' "$SCRATCH"/cong.1.out || fail "no 200 seen"
     # 32 requests times two 256 KiB rings is 16 MiB of ceiling; the
     # buffered path would have held 32 times the body instead.
     [ "$RSS_SAMPLES" -ge 3 ] || \
         fail "only $RSS_SAMPLES RSS samples taken across the concurrent transfers"
     rss_bound "$base" 32768 \
         "across 32 concurrent transfers of $((CONG_SMALL / 1024 / 1024)) MiB"
 
     # --- 14: not busy-waiting ---------------------------------------
     # An origin dribbling a byte at a time means paivana spends almost
     # the whole request with nothing to do.  If suspend/resume is wrong
     # -- an MHD content reader returning 0 without suspending, or an
     # unpause that reschedules itself -- the transfer still completes
     # and nothing else in the suite notices; a core is simply pinned for
     # the duration.  CPU time is the only thing that shows it.
     msg "an idle transfer costs no CPU, and its first byte still arrives at once"
     local c0 c1 jiffies
     c0="$(cpu_jiffies "$PAIVANA_PID")"
     "$BUILDDIR/stream_client" \
         "$(PAIVANA_URL "/cl?bytes=1024&rate=200")" \
         --expect-bytes 1024 > "$SCRATCH/sc" 2>&1 \
         || fail "stream_client did not finish"
     c1="$(cpu_jiffies "$PAIVANA_PID")"
     sc_is status 200
     sc_is pattern ok
     jiffies=$((c1 - c0))
     # ~5 s of wall clock, i.e. ~500 jiffies were available to burn.
     [ "$jiffies" -lt 100 ] || \
         fail "paivana used ${jiffies} jiffies of CPU across a ~5 s idle transfer; it is spinning"
     # And the point of streaming at all: the client does not wait for
     # the origin's last byte to see its first.
     [ "$(sfield ttfb_ms)" -lt 1000 ] || \
         fail "first byte took $(sfield ttfb_ms)ms of a ~5 s transfer; the body was buffered"
     echo "OK (${jiffies} jiffies CPU, first byte at $(sfield ttfb_ms)ms of $(sfield total_ms)ms)"
 
     # --- 15: pathological interleaving ------------------------------
     # A 1 KiB receive buffer makes libcurl drain paivana's socket in
     # tiny units, so MHD's content reader is called hundreds of times
     # for a body the default buffer would move in a handful -- and each
     # of those is a chance for the ring to empty and the connection to
     # suspend and resume.  Chunked, because that path re-enters MHD's
     # chunk framing on every one of them.
     msg "a client reading in 1 KiB units survives the pause/resume churn"
     "$BUILDDIR/stream_client" \
         "$(PAIVANA_URL "/chunked?bytes=$CONG_SMALL")" \
         --expect-bytes "$CONG_SMALL" --recv-buffer 1024 \
         > "$SCRATCH/sc" 2>&1 \
         || fail "stream_client did not finish"
     sc_is status 200
     sc_is chunked yes
     sc_is pattern ok
     sc_is bytes "$CONG_SMALL"
     ok
 
     # --- 16: slow at both ends simultaneously -----------------------
     # Neither side able to keep up with the other, on the same request.
     # Both rings spend the transfer alternately full and empty, and the
     # two halves of the state machine have to interleave without either
     # deadlocking or dropping a byte.
     msg "a slow upstream and a slow client on one request"
     "$BUILDDIR/stream_client" \
         "$(PAIVANA_URL "/cl?bytes=$CONG_SMALL&rate=$((8 * 1024 * 1024))")" \
         --expect-bytes "$CONG_SMALL" --read-rate $((4 * 1024 * 1024)) \
         > "$SCRATCH/sc" 2>&1 \
         || fail "stream_client did not finish"
     sc_is status 200
     sc_is pattern ok
     sc_is bytes "$CONG_SMALL"
     ok
 
     stop_paivana
 }
 
 
 function test_short_body() {
     # An incomplete message must be treated as a failure (RFC 9112
     # section 8.1.2), and with a streamed response the only way left to
     # say so is to break the framing.  The upstream's status and its
     # Content-Length reach the client long before the body runs out --
     # they cannot be retracted afterwards -- so the client sees a 200
     # promising 100000 bytes and a connection that closes after 1000.
     # curl reports that as error 18, which is the assertion here.
     #
     # A 502 used to be possible because the whole body was assembled
     # before anything was sent.  What must never happen either way is a
     # short body served as if it were complete: that is what the
     # "bytes remaining" check below rules out.
     msg "upstream body shorter than its Content-Length truncates the client"
     if ! command -v python3 >/dev/null 2>&1;
     then
         echo "SKIP (python3 missing)"
         return
     fi
     stop_paivana
     start_truncating_upstream "$TRUNC_PORT"
     start_paivana "http://127.0.0.1:$TRUNC_PORT"
     local out status
     # curl exits 18 (CURLE_PARTIAL_FILE) here, so the pipeline must not
     # be allowed to fail the test; the exit code is part of what is
     # being asserted.
     out="$(curl -sS -o "$SCRATCH/body" -D "$SCRATCH/hdr" \
                 -w '%{http_code}' --max-time 30 \
                 "$(PAIVANA_URL /short)" 2>"$SCRATCH/err")"
     status=$?
     [ "$status" = "18" ] || \
         fail "curl exit=$status want=18 (partial file); a 0 here means the truncation was hidden from the client"
     [ "$out" = "200" ] || \
         fail "status=$out want=200 (the upstream's status is already sent when the body runs out)"
     grep -qi '^Content-Length: 100000' "$SCRATCH/hdr" || \
         fail "client was not told the upstream's declared length: $(grep -i content-length "$SCRATCH/hdr")"
     [ "$(wc -c < "$SCRATCH/body")" = "1000" ] || \
         fail "got $(wc -c < "$SCRATCH/body") bytes, want the 1000 the upstream actually sent"
     ok
     stop_paivana
 }
 
 
 function test_upstream_down() {
     msg "upstream down yields 502 Bad Gateway"
     # Re-point paivana at a port with nothing listening.
     stop_paivana
     start_paivana "http://127.0.0.1:$DEAD_PORT"
     local status
     status="$(curl -sS -o "$SCRATCH/body" -w '%{http_code}' \
                    --max-time 10 \
                    "$(PAIVANA_URL /hello)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "502" ] || fail "status=$status want=502"
     grep -qi 'bad gateway' "$SCRATCH/body" || \
         fail "no 'Bad Gateway' in body"
     ok
 }
 
 # curl with multiple URLs on one command line uses HTTP keep-alive
 # (not true pipelining, but exercises the same code path in paivana
 # of handling successive requests on one TCP connection).
 function test_keepalive_curl() {
     msg "curl keep-alive: 3 sequential GETs on one connection"
     local out
     out="$(curl -sS --http1.1 \
                 -w '\n@status=%{http_code}\n' \
                 "$(PAIVANA_URL /hello)" \
                 "$(PAIVANA_URL /hello)" \
                 "$(PAIVANA_URL /hello)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local count
     count="$(printf '%s\n' "$out" | grep -c '^Hello from')"
     [ "$count" = "3" ] || fail "got $count Hello lines; want 3; out:"$'\n'"$out"
     ok
 }
 
 function test_wget_basic() {
     msg "wget fetch (third-party client interop)"
     if ! command -v wget >/dev/null 2>&1; then
         echo "SKIP (wget missing)"
         return
     fi
     local body
     body="$(wget -qO- --timeout=5 "$(PAIVANA_URL /hello)")" \
         || fail "wget failed"
     echo "$body" | grep -q '^Hello from' \
         || fail "unexpected body from wget: $body"
     ok
 }
 
 function test_pipelined() {
     msg "HTTP/1.1 pipelined requests (4 back-to-back on one TCP socket)"
     if [ ! -x "$PIPELINE_CLIENT" ]; then
         echo "SKIP (pipeline_client not built)"
         return
     fi
     local out
     out="$("$PIPELINE_CLIENT" 127.0.0.1 "$PAIVANA_PORT" \
               /hello /status/201 /hello /status/404 2>"$SCRATCH/err")" \
         || fail "pipeline_client: $(cat "$SCRATCH/err")"
     printf '%s\n' "$out" >"$SCRATCH/pipeline.out"
     local n
     n="$(grep -c '^--- response' "$SCRATCH/pipeline.out")"
     [ "$n" = "4" ] || fail "got $n responses, want 4; output:"$'\n'"$out"
     # Order preserved: responses must match the request sequence.
     grep -q '^--- response 0: status=200' "$SCRATCH/pipeline.out" \
         || fail "response 0: wrong status; out:"$'\n'"$out"
     grep -q '^--- response 1: status=201' "$SCRATCH/pipeline.out" \
         || fail "response 1: wrong status; out:"$'\n'"$out"
     grep -q '^--- response 2: status=200' "$SCRATCH/pipeline.out" \
         || fail "response 2: wrong status; out:"$'\n'"$out"
     grep -q '^--- response 3: status=404' "$SCRATCH/pipeline.out" \
         || fail "response 3: wrong status; out:"$'\n'"$out"
     ok
 }
 
 ######################################################################
 # Forwarding headers (X-Forwarded-*), with and without -f.
 #
 # Which of two roles paivana plays is decided by -f, the same flag
 # that decides where the access cookie's client address comes from:
 # without it we are the outermost proxy and a client's assertions are
 # replaced with what we can see; with it we are behind a trusted proxy
 # and extend the chain it gave us.  Both directions are asserted here,
 # because getting either wrong is silent -- the request still
 # succeeds, it just carries the wrong client.
 ######################################################################
 
 # Echo the upstream's view of one header.  $1 = header name.
 function upstream_header() {
     grep -i "^$1:" "$SCRATCH/body" | tr -d '\r' | sed -e "s/^[^:]*: *//"
 }
 
 function test_forwarded_no_flag() {
     msg "no -f: client X-Forwarded-* are replaced, not believed"
     curl -sS -H 'X-Forwarded-For: 1.2.3.4' \
          -H 'X-Forwarded-Proto: https' \
          -H 'X-Forwarded-Host: evil.example.com' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local xff proto host
     xff="$(upstream_header x-forwarded-for)"
     proto="$(upstream_header x-forwarded-proto)"
     host="$(upstream_header x-forwarded-host)"
     [ "$xff" = "127.0.0.1" ] || \
         fail "X-Forwarded-For='$xff', want '127.0.0.1' (client's 1.2.3.4 must not survive)"
     # The client asserted https.  We are plain HTTP, and without -f
     # nothing the client says about the scheme may be believed --
     # otherwise it picks the scheme of the URLs we generate for it.
     [ "$proto" = "http" ] || \
         fail "X-Forwarded-Proto='$proto', want 'http' (client asserted https)"
     case "$host" in
         *evil.example.com*) fail "client's X-Forwarded-Host reached upstream: '$host'";;
     esac
     ok
 }
 
 function test_forwarded_with_flag() {
     msg "-f: inbound chain is extended, not discarded"
     stop_paivana
     start_paivana "$PAIVANA_DEST" -f
     curl -sS -H 'X-Forwarded-For: 203.0.113.7, 198.51.100.9' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local xff
     xff="$(upstream_header x-forwarded-for)"
     # Our own peer is appended to the right of the chain we were given.
     [ "$xff" = "203.0.113.7, 198.51.100.9, 127.0.0.1" ] || \
         fail "X-Forwarded-For='$xff', want '203.0.113.7, 198.51.100.9, 127.0.0.1'"
     ok
 
     msg "-f: trusted X-Forwarded-Proto / -Host are passed through"
     curl -sS -H 'X-Forwarded-Proto: https' \
          -H 'X-Forwarded-Host: public.example.com' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local proto host
     proto="$(upstream_header x-forwarded-proto)"
     host="$(upstream_header x-forwarded-host)"
     [ "$proto" = "https" ] || \
         fail "X-Forwarded-Proto='$proto', want 'https' (trusted proxy said so)"
     [ "$host" = "public.example.com" ] || \
         fail "X-Forwarded-Host='$host', want 'public.example.com'"
     ok
 
     msg "-f: no inbound chain still yields our own peer"
     curl -sS -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     xff="$(upstream_header x-forwarded-for)"
     [ "$xff" = "127.0.0.1" ] || \
         fail "X-Forwarded-For='$xff', want '127.0.0.1'"
     ok
 
     msg "-f: a repeated X-Forwarded-For is combined into one chain"
     # RFC 9110 §5.3: two field lines of a list header mean the same as
     # one comma-joined line, and must reach the origin as one header.
     curl -sS -H 'X-Forwarded-For: 203.0.113.7' \
          -H 'X-Forwarded-For: 198.51.100.9' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local n
     n="$(grep -ci '^x-forwarded-for:' "$SCRATCH/body")"
     [ "$n" = "1" ] || fail "upstream saw $n X-Forwarded-For headers, want 1"
     xff="$(upstream_header x-forwarded-for)"
     [ "$xff" = "203.0.113.7, 198.51.100.9, 127.0.0.1" ] || \
         fail "X-Forwarded-For='$xff', want '203.0.113.7, 198.51.100.9, 127.0.0.1'"
     ok
 
     stop_paivana
     start_paivana "$PAIVANA_DEST"
 }
 
 function test_forwarded_unix() {
     # The deployment the Debian packaging actually ships: paivana on a
     # Unix socket behind nginx/Apache.  A Unix peer has no address, so
     # without -f there is nothing to put in X-Forwarded-For at all, and
     # with -f the inbound chain is the only client information that
     # exists -- losing it leaves the origin blind.
     msg "unix socket, -f: inbound chain survives the address-less hop"
     local dest="$PAIVANA_DEST"
     stop_paivana
     start_paivana_unix "$dest" -f
     curl -sS --unix-socket "$PAIVANA_SOCK" \
          -H 'X-Forwarded-For: 203.0.113.7' \
          -o "$SCRATCH/body" http://localhost/echo-headers 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local xff
     xff="$(upstream_header x-forwarded-for)"
     # Nothing is appended: a Unix peer has no address, and inventing
     # one ("127.0.0.1") would be indistinguishable from a real
     # loopback client.  The hop is recorded in Via instead.
     [ "$xff" = "203.0.113.7" ] || \
         fail "X-Forwarded-For='$xff', want '203.0.113.7' (unadorned)"
     grep -qi '^via:.*paivana' "$SCRATCH/body" || \
         fail "Via does not record the paivana hop; headers:"$'\n'"$(cat "$SCRATCH/body")"
     ok
 
     msg "unix socket, no -f: no X-Forwarded-For is invented"
     stop_paivana
     start_paivana_unix "$dest"
     curl -sS --unix-socket "$PAIVANA_SOCK" \
          -H 'X-Forwarded-For: 1.2.3.4' \
          -o "$SCRATCH/body" http://localhost/echo-headers 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     grep -qi '^x-forwarded-for:' "$SCRATCH/body" && \
         fail "upstream saw an X-Forwarded-For we cannot substantiate:"$'\n'"$(cat "$SCRATCH/body")"
     grep -qi '^via:.*paivana' "$SCRATCH/body" || \
         fail "Via does not record the paivana hop"
     ok
 
     stop_paivana
     start_paivana "$dest"
 }
 
 function test_forwarded_rfc7239() {
     stop_paivana
     start_paivana "$PAIVANA_DEST" -f
 
     msg "-f: RFC 7239 Forwarded is extended with our own element"
     curl -sS -H 'Forwarded: for=203.0.113.7;proto=https;host=public.example.com' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local fwd
     fwd="$(upstream_header forwarded)"
     case "$fwd" in
         "for=203.0.113.7;proto=https;host=public.example.com, for=127.0.0.1;by=_paivana;"*) ;;
         *) fail "Forwarded not extended correctly: '$fwd'";;
     esac
     ok
 
     # A proxy that speaks only RFC 7239 must still be understood by an
     # origin that speaks only X-Forwarded-*, or we would report the
     # proxy as the client and the wrong scheme with it.
     msg "-f: Forwarded is mirrored into the X-Forwarded-* headers"
     local xff proto host
     xff="$(upstream_header x-forwarded-for)"
     proto="$(upstream_header x-forwarded-proto)"
     host="$(upstream_header x-forwarded-host)"
     [ "$xff" = "203.0.113.7, 127.0.0.1" ] || \
         fail "X-Forwarded-For='$xff', want '203.0.113.7, 127.0.0.1'"
     [ "$proto" = "https" ] || \
         fail "X-Forwarded-Proto='$proto', want 'https' (Forwarded said so)"
     [ "$host" = "public.example.com" ] || \
         fail "X-Forwarded-Host='$host', want 'public.example.com'"
     ok
 
     # RFC 7239 §6.3: "unknown" is a legal node identifier, but
     # X-Forwarded-For has no way to say it -- so no chain is
     # synthesized rather than one with a hop silently missing.
     msg "-f: a Forwarded chain that X-Forwarded-For cannot express is not faked"
     curl -sS -H 'Forwarded: for=unknown' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     xff="$(upstream_header x-forwarded-for)"
     [ "$xff" = "127.0.0.1" ] || \
         fail "X-Forwarded-For='$xff', want just our own peer '127.0.0.1'"
     fwd="$(upstream_header forwarded)"
     case "$fwd" in
         "for=unknown, for=127.0.0.1;by=_paivana;"*) ;;
         *) fail "Forwarded should still carry the unknown hop: '$fwd'";;
     esac
     ok
 
     msg "no -f: a client's Forwarded is replaced, not extended"
     stop_paivana
     start_paivana "$PAIVANA_DEST"
     curl -sS -H 'Forwarded: for=1.2.3.4;proto=https' \
          -o "$SCRATCH/body" "$(PAIVANA_URL /echo-headers)" 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     fwd="$(upstream_header forwarded)"
     case "$fwd" in
         *1.2.3.4*) fail "client's Forwarded element survived: '$fwd'";;
     esac
     case "$fwd" in
         "for=127.0.0.1;by=_paivana;proto=http;"*) ;;
         *) fail "unexpected Forwarded: '$fwd'";;
     esac
     ok
 }
 
 function test_forwarded_unix_rfc7239() {
     # Unlike X-Forwarded-For, RFC 7239 has a spelling for a hop with no
     # address (§6.3 "unknown"), so the Unix hop need not go unrecorded.
     msg "unix socket: our Forwarded element says for=unknown"
     local dest="$PAIVANA_DEST"
     stop_paivana
     start_paivana_unix "$dest" -f
     curl -sS --unix-socket "$PAIVANA_SOCK" \
          -H 'Forwarded: for=203.0.113.7' \
          -o "$SCRATCH/body" http://localhost/echo-headers 2>"$SCRATCH/err" \
         || fail "curl: $(cat "$SCRATCH/err")"
     local fwd
     fwd="$(upstream_header forwarded)"
     case "$fwd" in
         "for=203.0.113.7, for=unknown;by=_paivana;"*) ;;
         *) fail "unexpected Forwarded over a Unix socket: '$fwd'";;
     esac
     ok
     stop_paivana
     start_paivana "$dest"
 }
 
 ######################################################################
 # TRUSTED_PROXIES configuration validation.
 #
 # The GNUnet policy parsers accept several things that mean "nothing
 # usable" without saying so -- a missing terminator, a /0 network
 # (which is indistinguishable from the list terminator), an address of
 # the wrong family.  Quietly trusting nobody would send every visitor
 # to the socket address with no hint why, so the loader refuses to
 # start instead.  These cases pin that.
 #
 # Worse than "nothing usable" is "some of it": a list whose LAST entry
 # has no ';' loses that entry and reports success for the rest, so a
 # single typo would leave every client behind the unlisted proxy
 # sharing that proxy's address -- and one paid cookie.  The loader
 # counts the entries it got back against the ';' that went in.
 ######################################################################
 
 # Start paivana with an extra config line and report whether it came
 # up.  Echoes "started" or "refused".
 #
 # "Refused" is read off wait_for_port giving up, so the pid has to be
 # passed: a refused config makes paivana exit in milliseconds, and
 # without the liveness check the verdict would come from a five-second
 # timeout instead -- and would be wrong outright if anything else were
 # holding the port, since the loop would then see a listener and call
 # every refusal an acceptance.
 function paivana_with_config_line() {
     local line="$1"
     local nofile="${2:-}"
     local cfg="$SCRATCH/startup.conf"
     sed -e "s|@DEST@|http://127.0.0.1:$MHD_PORT|g" \
         -e "s|@PORT@|$PAIVANA_PORT|g" \
         "$SRCDIR/test_reverse_proxy.conf.in" > "$cfg"
     printf '%s\n' "$line" >> "$cfg"
     local log="$LOGDIR/startup.log"
     if [ -n "$nofile" ];
     then
         ( ulimit -n "$nofile"; exec "$PAIVANA_HTTPD" -c "$cfg" -n -f -L ERROR ) >"$log" 2>&1 &
     else
         ( exec "$PAIVANA_HTTPD" -c "$cfg" -n -f -L ERROR ) >"$log" 2>&1 &
     fi
     local pid=$!
     if wait_for_port 127.0.0.1 "$PAIVANA_PORT" "$pid";
     then
         kill -TERM "$pid" 2>/dev/null
         wait "$pid" 2>/dev/null
         echo "started"
         return
     fi
     kill -TERM "$pid" 2>/dev/null
     wait "$pid" 2>/dev/null
     echo "refused"
 }
 
 function test_resource_budget_config() {
     stop_paivana
     local r
 
     # The default ring calculation is exactly
     # 352 * (262144 + 262144) = 184549376 bytes.  Testing one byte on
     # either side pins the overflow-safe aggregate comparison.
     for bad in \
         'CONNECTION_LIMIT = 385' \
         'PAYMENT_CONNECTION_LIMIT = 0' \
         'CONNECTION_LIMIT = 32' \
         'RELAY_MEMORY_LIMIT = 184549375'
     do
         msg "startup refused: $bad"
         r="$(paivana_with_config_line "$bad")"
         [ "$r" = "refused" ] || \
             fail "paivana started with an unsafe resource budget ($bad)"
         ok
     done
 
     for good in \
         'CONNECTION_LIMIT = 384' \
         'PAYMENT_CONNECTION_LIMIT = 32' \
         'RELAY_MEMORY_LIMIT = 184549376' \
         'SHUTDOWN_GRACE_PERIOD = 0 s'
     do
         msg "startup accepted: $good"
         r="$(paivana_with_config_line "$good")"
         [ "$r" = "started" ] || \
             fail "paivana refused a safe resource budget ($good); log:"$'\n'"$(cat "$LOGDIR/startup.log")"
         ok
     done
 
     # CONNECTION_LIMIT=10 charges 20 descriptors plus the documented
     # 256-descriptor reserve.  The soft limit is a hard startup boundary,
     # independently of the much larger select() ceiling.
     msg "startup refused one descriptor below its calculated requirement"
     r="$(paivana_with_config_line \
         $'CONNECTION_LIMIT = 10\nPAYMENT_CONNECTION_LIMIT = 2' 275)"
     [ "$r" = "refused" ] || fail "paivana ignored RLIMIT_NOFILE=275"
     ok
 
     msg "startup accepted at its exact calculated descriptor requirement"
     r="$(paivana_with_config_line \
         $'CONNECTION_LIMIT = 10\nPAYMENT_CONNECTION_LIMIT = 2' 276)"
     [ "$r" = "started" ] || \
         fail "paivana refused RLIMIT_NOFILE=276; log:"$'\n'"$(cat "$LOGDIR/startup.log")"
     ok
 
     start_paivana "http://127.0.0.1:$MHD_PORT"
 }
 
 function test_controlled_overload() {
     stop_paivana
     start_paivana_with_config "http://127.0.0.1:$MHD_PORT" \
         $'CONNECTION_LIMIT = 4\nPAYMENT_CONNECTION_LIMIT = 1'
 
     # Three ordinary requests are the entire non-payment budget.  They remain
     # suspended in Paivana while the single-threaded test origin answers its
     # slow requests, leaving the fourth MHD connection for an immediate
     # overload response and then for the reserved payment request.
     local pids=()
     for i in 1 2 3;
     do
         curl -sS -o /dev/null -w '%{http_code}' \
             "$(PAIVANA_URL /slow/1000)" >"$SCRATCH/ordinary-$i.status" &
         pids+=("$!")
     done
     sleep 0.3
 
     msg "ordinary capacity returns a controlled 503"
     local status
     status="$(curl -sS --max-time 2 -D "$SCRATCH/overload.headers" \
         -o "$SCRATCH/overload.body" -w '%{http_code}' \
         "$(PAIVANA_URL /small)")" || fail "ordinary overload request failed"
     [ "$status" = 503 ] || fail "ordinary overload status=$status, want 503"
     grep -qi '^Retry-After: 1' "$SCRATCH/overload.headers" || \
         fail "ordinary overload response lacks Retry-After: 1"
     grep -qi '^Connection: close' "$SCRATCH/overload.headers" || \
         fail "ordinary overload response lacks Connection: close"
     ok
 
     msg "the reserved slot still admits the payment endpoint"
     status="$(curl -sS --max-time 2 -o "$SCRATCH/payment-disabled.body" \
         -w '%{http_code}' -H 'Content-Type: application/json' -X POST \
         -d '{}' "$(PAIVANA_URL /.well-known/paivana)")" || \
         fail "reserved payment request failed"
     [ "$status" = 501 ] || \
         fail "reserved payment status=$status, want the -n response 501"
     ok
 
     for i in 0 1 2;
     do
         wait "${pids[$i]}" || fail "ordinary request $((i + 1)) failed"
         [ "$(cat "$SCRATCH/ordinary-$((i + 1)).status")" = 200 ] || \
             fail "ordinary request $((i + 1)) did not complete"
     done
     stop_paivana
     start_paivana "http://127.0.0.1:$MHD_PORT"
 }
 
 function test_graceful_shutdown() {
     stop_paivana
     start_paivana_with_config "http://127.0.0.1:$MHD_PORT" \
         $'CONNECTION_LIMIT = 10\nPAYMENT_CONNECTION_LIMIT = 2\nSHUTDOWN_GRACE_PERIOD = 2 s' INFO
 
     curl -sS -o /dev/null -w '%{http_code}' \
         "$(PAIVANA_URL /slow/1000)" >"$SCRATCH/drain.status" &
     local curl_pid=$!
     sleep 0.2
     msg "SIGTERM drains an accepted request within the grace period"
     kill -TERM "$PAIVANA_PID"
     wait "$curl_pid" || fail "request was dropped during graceful shutdown"
     [ "$(cat "$SCRATCH/drain.status")" = 200 ] || \
         fail "drained request did not retain its 200 response"
     wait "$PAIVANA_PID" || fail "paivana exited unsuccessfully after draining"
     PAIVANA_PID=""
     grep -q 'Graceful shutdown drained all requests' \
         "$LOGDIR/paivana-custom.log" || \
         fail "graceful-drain completion was not logged"
     ok
 
     start_paivana_with_config "http://127.0.0.1:$MHD_PORT" \
         $'CONNECTION_LIMIT = 10\nPAYMENT_CONNECTION_LIMIT = 2\nSHUTDOWN_GRACE_PERIOD = 100 ms' INFO
 
     curl -sS -o /dev/null "$(PAIVANA_URL /slow/2000)" &
     curl_pid=$!
     sleep 0.2
     msg "the shutdown deadline cancels a suspended request without aborting"
     kill -TERM "$PAIVANA_PID"
     # The forced shutdown deliberately terminates this client request.  Its
     # result is immaterial; the daemon must exit normally rather than letting
     # MHD abort because the connection was still suspended.
     wait "$curl_pid" || :
     local paivana_status
     if wait "$PAIVANA_PID";
     then
         paivana_status=0
     else
         paivana_status=$?
     fi
     PAIVANA_PID=""
     [ "$paivana_status" = 0 ] || \
         fail "paivana exited with status $paivana_status at the drain deadline"
     grep -q 'Graceful shutdown deadline reached with 1 active request' \
         "$LOGDIR/paivana-custom.log" || \
         fail "graceful-drain deadline was not reached with an active request"
     ok
 
     start_paivana "http://127.0.0.1:$MHD_PORT"
 }
 
 function test_trusted_proxies_config() {
     stop_paivana
     local r
 
     # Bad values must be refused loudly rather than silently ignored.
     for bad in \
         'TRUSTED_PROXIES = 10.0.0.0/8' \
         'TRUSTED_PROXIES = 0.0.0.0/0;' \
         'TRUSTED_PROXIES = ::1;' \
         'TRUSTED_PROXIES = garbage;' \
         'TRUSTED_PROXIES = 10.0.0.0/8;192.168.0.0/16' \
         'TRUSTED_PROXIES = 10.0.0.0/8;garbage' \
         'TRUSTED_PROXIES6 = 2001:db8::/32' \
         'TRUSTED_PROXIES6 = 2001:db8::/32;fe80::/10' \
         'TRUSTED_PROXIES6 = ::/0;' \
         'TRUSTED_PROXIES6 = 2001:db8::/32; fe80::/10;'
     do
         msg "startup refused: $bad"
         r="$(paivana_with_config_line "$bad")"
         [ "$r" = "refused" ] || \
             fail "paivana started with an unusable policy ($bad)"
         ok
     done
 
     # ...and good ones must of course still start.
     for good in \
         'TRUSTED_PROXIES = 10.0.0.0/8;192.168.0.0/16;' \
         'TRUSTED_PROXIES = 127.0.0.1;' \
         'TRUSTED_PROXIES6 = 2001:db8::/32;fe80::/10;' \
         'TRUSTED_PROXIES6 = ::1;'
     do
         msg "startup accepted: $good"
         r="$(paivana_with_config_line "$good")"
         [ "$r" = "started" ] || \
             fail "paivana refused a usable policy ($good); log:"$'\n'"$(cat "$LOGDIR/startup.log")"
         ok
     done
 
     start_paivana "http://127.0.0.1:$MHD_PORT"
 }
 
 ######################################################################
 # WHITELIST configuration validation.
 #
 # WHITELIST names the paths served without payment, and paivana wraps
 # it in "^(%s)$" before regcomp: regexec(3) is unanchored, so a
 # WHITELIST of "/free/" would otherwise waive payment for every URL
 # merely *containing* it, and the group keeps an alternation from
 # binding the anchors to only its outer branches.
 #
 # What the suite can reach of this is the loading, not the matching:
 # regcomp happens at config time regardless of -n, while the regexec
 # sits behind the paywall that -n switches off, and paivana will not
 # start without -n unless a merchant backend is there to serve it
 # templates.  So these cases pin that an unusable expression is
 # refused rather than carried into the process -- the alternative
 # being a paivana that runs with an uninitialised regex_t.
 #
 # The last two bad values are the anchoring itself, as far as it can
 # be reached from here.  "a)|(b" and "(a$|^b" do not balance on their
 # own, so wrapping them yields "^(a)|(b)$" and "^((a$|^b))$" -- an
 # alternation that has climbed out of the group, with one branch
 # anchored on one side only and the whitelist consequently matching
 # far more than it says.  paivana compiles the value bare first for
 # exactly this reason, so the configuration is refused rather than
 # accepted into a regex that means something else.
 ######################################################################
 
 function test_whitelist_config() {
     stop_paivana
     local r
 
     for bad in \
         'WHITELIST = *invalid(' \
         'WHITELIST = /free/[' \
         'WHITELIST = /free/\' \
         'WHITELIST = a)|(b' \
         'WHITELIST = (a$|^b'
     do
         msg "startup refused: $bad"
         r="$(paivana_with_config_line "$bad")"
         [ "$r" = "refused" ] || \
             fail "paivana started with an uncompilable WHITELIST ($bad)"
         ok
     done
 
     for good in \
         'WHITELIST = /free/.*' \
         'WHITELIST = /free/.*|/assets/.*' \
         'WHITELIST = ^/free/.*$'
     do
         msg "startup accepted: $good"
         r="$(paivana_with_config_line "$good")"
         [ "$r" = "started" ] || \
             fail "paivana refused a usable WHITELIST ($good); log:"$'\n'"$(cat "$LOGDIR/startup.log")"
         ok
     done
 
     start_paivana "http://127.0.0.1:$MHD_PORT"
 }
 
 ######################################################################
 # The payment endpoint under -n.
 #
 # POST /.well-known/paivana is the one paywall-side branch that -n
 # does not shield: the handler answers 501 rather than falling through
 # to the proxy.  Everything else about the endpoint is unreachable
 # here, but this much is worth pinning, because the two ways to get it
 # wrong are both silent.  Forwarding the POST upstream would hand the
 # origin a request carrying payment data it has no business seeing;
 # claiming the path for every method would shadow whatever the origin
 # serves at that URL.
 ######################################################################
 
 function test_paywall_disabled_endpoint() {
     msg "-n: POST /.well-known/paivana is 501, not forwarded"
     local status
     status="$(curl -sS -D "$SCRATCH/hdrs" -o "$SCRATCH/body" -w '%{http_code}' \
                    -X POST -H 'Content-Type: application/json' \
                    --data '{}' \
                    "$(PAIVANA_URL /.well-known/paivana)" \
                    2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" = "501" ] || fail "status=$status want=501"
     # An upstream that had seen the request would have labelled the
     # response; paivana answering for itself does not.
     if grep -qi '^x-upstream:' "$SCRATCH/hdrs";
     then
         fail "the POST reached the upstream"
     fi
     ok
 
     # `is_paivana' is set only for POST, so the endpoint must not
     # swallow the origin's own URL space at that path.
     msg "-n: GET /.well-known/paivana is forwarded like any other path"
     status="$(curl -sS -D "$SCRATCH/hdrs" -o "$SCRATCH/body" -w '%{http_code}' \
                    "$(PAIVANA_URL /.well-known/paivana)" 2>"$SCRATCH/err")" \
         || fail "curl: $(cat "$SCRATCH/err")"
     [ "$status" != "501" ] || fail "GET was answered by the paywall handler"
     grep -qi '^x-upstream:' "$SCRATCH/hdrs" || \
         fail "GET did not reach the upstream (status=$status)"
     ok
 }
 
 ######################################################################
 # Drive the tests.
 ######################################################################
 
 echo "=== paivana reverse-proxy tests ==="
 echo "Temp dir: $SCRATCH"
 echo "Paivana binary: $PAIVANA_HTTPD"
 echo "Source dir: $SRCDIR"
 echo "Build dir: $BUILDDIR"
 echo "Ports: $PORT_BASE + 1..8, 99, 100"
 
 require_ports_free "$MHD_PORT" "$GO_PORT" "$PY_PORT" "$RS_PORT" \
                    "$EARLY_PORT" "$NODRAIN_PORT" "$TRUNC_PORT" \
                    "$STREAM_PORT" "$DEAD_PORT" "$PAIVANA_PORT"
 
 start_upstreams
 
 # --- C / libmicrohttpd upstream ---------------------------------------
 start_paivana "http://127.0.0.1:$MHD_PORT"
 run_battery "mhd"
 
 test_method_not_allowed
 test_upload_too_big
 test_upload_too_big_early
 test_upload_too_big_no_continue
 test_upload_too_big_chunked
 test_keepalive_curl
 test_wget_basic
 test_pipelined
 test_forwarded_no_flag
 test_forwarded_with_flag
 test_forwarded_unix
 test_forwarded_rfc7239
 test_forwarded_unix_rfc7239
 test_paywall_disabled_endpoint
 test_resource_budget_config
 test_controlled_overload
 test_graceful_shutdown
 test_trusted_proxies_config
 test_whitelist_config
 
 stop_paivana
 
 # --- Go upstream ------------------------------------------------------
 if [ -n "$GO_PORT" ];
 then
     start_paivana "http://127.0.0.1:$GO_PORT"
     run_battery "go"
     test_pipelined
     stop_paivana
 fi
 
 # --- Python upstream --------------------------------------------------
 if [ -n "$PY_PORT" ];
 then
     start_paivana "http://127.0.0.1:$PY_PORT"
     run_battery "py"
     test_pipelined
     stop_paivana
 fi
 
 # --- Rust upstream ----------------------------------------------------
 if [ -n "$RS_PORT" ];
 then
     start_paivana "http://127.0.0.1:$RS_PORT"
     run_battery "rs"
     test_pipelined
     stop_paivana
 fi
 
 # --- Early-response upstream tests (restart paivana) ------------------
 test_early_response
 test_early_response_no_drain
 
 # --- Streaming (restarts paivana) -------------------------------------
 test_streaming
 
 # --- Congestion and the memory bound (restarts paivana) ---------------
 test_congestion
 
 # --- Truncated-response test (restarts paivana) -----------------------
 test_short_body
 
 # --- Upstream-down test (runs last because it restarts paivana) -------
 test_upstream_down
 stop_paivana
 
 echo "=== all tests passed ==="
 exit 0