commit 36e60f229af05a2a5787c7e705f19c29c5d12740
parent 66e4c7754568e545591102129322e9b75ff051d4
Author: Florian Dold <dold@taler.net>
Date: Fri, 11 Sep 2026 21:29:11 +0200
deployment: reapply service configuration after interrupted runs
Keep an incomplete-deployment marker until handlers and sanity checks
succeed. On retries, requeue nginx and monitoring handlers so unchanged
files from a failed run are applied to the running services.
Validate Fluent Bit before restarting and require its enabled latency
endpoint to respond before monitoring setup succeeds.
Diffstat:
11 files changed, 344 insertions(+), 2 deletions(-)
diff --git a/README b/README
@@ -84,6 +84,15 @@ If you are root@rusty.taler-ops.ch, you may be able to:
$ ./deploy rusty
```
+Deployments keep `/var/lib/taler-deployment/incomplete` until configuration
+handlers and post-deployment checks have succeeded. If a run fails or is
+interrupted, the next `deploy` reapplies nginx and enabled monitoring service
+configuration even when the files are unchanged. This recovers restart
+notifications lost by the previous Ansible process. Applications still stay
+stopped on deployment failure; check mode neither clears the marker nor
+restarts services. Normal successful reruns keep the usual change-triggered
+handler behavior.
+
For TOPS production, replace the "rusty" with "spec" to use the actual secrets
for the deployment. For this, you first need to decrypt them:
@@ -554,6 +563,14 @@ Fluent Bit binary's `--dry-run` mode and nginx's `-t` mode.
## Regression checks
+`python3 contrib/tests/test_deployment_recovery.py` uses the local
+`ansible-taler-test` image built by `./test` (or a Debian systemd image passed
+with `--image`). It runs the real setup failure/retry flow and recovery handlers
+against disposable service fixtures, checking lost notifications, unchanged
+retries, check mode, validation failures and disabled monitoring. It requires
+the `containers.podman` Ansible collection and runs without container networking
+or package downloads. The full `./test` runner includes this regression.
+
`python3 contrib/tests/test_nginx_latency.py` checks the timing parser and bounded
route classification (requires Python Jinja2 and Lua). The `tsys-infra/monitoring`
Podman suite exercises this repository's real monitoring role, nginx GET
diff --git a/contrib/tests/test_deployment_recovery.py b/contrib/tests/test_deployment_recovery.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Exercise setup's failure/retry flow with disposable systemd service fixtures.
+
+Uses the real setup play, deployment state, recovery tasks and service handlers.
+Package installation and application configuration are replaced by small roles
+so the regression needs no Taler deployment, certificates or package downloads.
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import time
+
+import yaml
+
+REPO = Path(__file__).resolve().parents[2]
+INFRASTRUCTURE = ['nginx', 'prometheus-node-exporter', 'node-exporter-proxy', 'fluent-bit']
+
+
+def run(*args, **kwargs):
+ return subprocess.run(list(map(str, args)), capture_output=True, text=True,
+ check=kwargs.pop('check', True), **kwargs)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--image', default='localhost/ansible-taler-test:latest',
+ help='Local Debian image with systemd, Python and dbus installed')
+ args = parser.parse_args()
+ with tempfile.TemporaryDirectory(prefix='taler-deployment-recovery-') as directory:
+ work = Path(directory)
+ container_name = 'taler-deployment-recovery-' + str(os.getpid())
+
+ def container(*cmd, **kwargs):
+ return run('podman', 'exec', '-i', container_name, *cmd, **kwargs)
+
+ def write(path, value):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(yaml.safe_dump(value, sort_keys=False))
+
+ def role(name, tasks):
+ write(work / 'roles' / name / 'tasks/main.yml', tasks)
+
+ def production_tasks(role_name):
+ return yaml.safe_load((REPO / 'roles' / role_name / 'tasks/main.yml').read_text())
+
+ def fail(stage):
+ return {'name': 'Inject deployment failure at ' + stage,
+ 'ansible.builtin.fail': {'msg': 'Injected regression failure'},
+ 'when': f"failure_stage == '{stage}'"}
+
+ shutil.copyfile(REPO / 'playbooks/setup.yml', work / 'setup.yml')
+ shutil.copytree(REPO / 'roles/deployment_state', work / 'roles/deployment_state')
+ for name in ['common_packages', 'database', 'libeufin-nexus', 'challenger',
+ 'exchange', 'auditor', 'devtesting']:
+ role(name, [])
+ role('stop_services', [
+ {'ansible.builtin.systemd_service': {'name': 'taler-exchange-httpd', 'state': 'stopped'}},
+ {'ansible.builtin.set_fact': {'stop_services_active_units': []}}])
+ role('start_services', [
+ {'ansible.builtin.systemd_service': {'name': 'taler-exchange-httpd', 'state': 'started'}}])
+ role('post_deployment_checks', [fail('sanity-checks')])
+ nginx_tasks = production_tasks('webserver')
+ role('webserver', [
+ {'ansible.builtin.copy': {'content': '{{ revision }}\n',
+ 'dest': '/etc/nginx/nginx.conf', 'mode': '0644'},
+ 'notify': 'Restart nginx'},
+ *nginx_tasks[-2:]]) # Real validation/recovery notification and enablement.
+ for name in ['webserver', 'monitoring']:
+ shutil.copytree(REPO / 'roles' / name / 'handlers', work / 'roles' / name / 'handlers')
+ monitor_tasks = production_tasks('monitoring')
+ apply_index = next(i for i, task in enumerate(monitor_tasks)
+ if task['name'] == 'Load the configured monitoring units')
+ role('monitoring', [
+ monitor_tasks[1], # Service discovery for check mode.
+ {'ansible.builtin.copy': {'content': '{{ revision }}\n',
+ 'dest': '{{ item }}', 'mode': '0600'},
+ 'loop': ['/etc/fluent-bit/fluent-bit.conf', '/etc/taler-monitoring/nginx.conf',
+ '/etc/default/prometheus-node-exporter'],
+ 'notify': ['Restart monitoring Fluent Bit', 'Restart monitoring proxy',
+ 'Restart monitoring node_exporter', 'Restart monitoring journal']},
+ fail('before-handlers'), *monitor_tasks[apply_index:]])
+ shutil.copyfile(REPO / 'roles/monitoring/tasks/recover.yml',
+ work / 'roles/monitoring/tasks/recover.yml')
+ write(work / 'roles/monitoring/tasks/preflight.yml', [])
+ write(work / 'inventory.yml', {'all': {'hosts': {container_name: {
+ 'ansible_connection': 'containers.podman.podman', 'ansible_user': 'root',
+ 'ansible_python_interpreter': '/usr/bin/python3'}}}})
+ (work / 'ansible.cfg').write_text('[defaults]\nroles_path = ' + str(work / 'roles') + '\n')
+ environment = dict(os.environ, ANSIBLE_CONFIG=str(work / 'ansible.cfg'), ANSIBLE_NOCOWS='1')
+
+ def deploy(revision=1, failure_stage='', check=False, enabled=True, expect_failure=False):
+ variables = dict(revision=revision, failure_stage=failure_stage,
+ enable_monitoring=enabled, monitoring_nginx_latency_enabled=revision >= 2,
+ deployment_kind='regression', exchange_attribute_encryption_key='fixture',
+ devtesting_mock_mfa=False, deploy_auditor=False, deploy_challenger=False,
+ use_ebics=False, configure_ebics=False)
+ write(work / 'vars.yml', variables)
+ command = ['ansible-playbook', '-i', work / 'inventory.yml', work / 'setup.yml',
+ '-e', '@' + str(work / 'vars.yml')]
+ if check:
+ command.append('--check')
+ result = run(*command, env=environment, check=False)
+ if bool(result.returncode) != expect_failure:
+ raise AssertionError(result.stdout + result.stderr)
+ return result.stdout
+
+ def pids():
+ values = container('systemctl', 'show', '--value', '-p', 'MainPID', *INFRASTRUCTURE).stdout.split()
+ return dict(zip(INFRASTRUCTURE, values, strict=True))
+
+ def marker():
+ return container('test', '-f', '/var/lib/taler-deployment/incomplete', check=False).returncode == 0
+
+ def app_active():
+ return container('systemctl', 'is-active', '--quiet', 'taler-exchange-httpd', check=False).returncode == 0
+
+ def listener():
+ return container('python3', '-c',
+ 'import urllib.request; print(urllib.request.urlopen('
+ '"http://127.0.0.1:2021/metrics", timeout=2).status)', check=False).returncode == 0
+
+ run('podman', 'run', '--detach', '--name', container_name, '--network', 'none',
+ '--systemd', 'always', '--tmpfs', '/run', '--tmpfs', '/tmp',
+ args.image, '/sbin/init')
+ try:
+ for _ in range(30):
+ state = container('systemctl', 'is-system-running', check=False).stdout.strip()
+ if state in ['running', 'degraded']:
+ break
+ time.sleep(1)
+ else:
+ raise AssertionError('Test systemd did not start')
+ # Each fixture reads its configuration at startup. Fluent Bit's
+ # second revision opens the latency listener, just as on rusty.
+ service = work / 'service.py'
+ service.write_text('''#!/usr/bin/python3
+import http.server, pathlib, sys, time
+name, config = sys.argv[1:]
+revision = pathlib.Path(config).read_text().strip()
+if name == 'fluent-bit' and int(revision) >= 2 and not pathlib.Path('/tmp/disable-metrics').exists():
+ class Metrics(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(200)
+ self.end_headers()
+ def log_message(self, *args):
+ pass
+ http.server.HTTPServer(('127.0.0.1', 2021), Metrics).serve_forever()
+else:
+ time.sleep(3600)
+''')
+ container('mkdir', '-p', '/etc/nginx', '/etc/taler-monitoring', '/etc/fluent-bit',
+ '/opt/fluent-bit/bin')
+ run('podman', 'cp', service, f'{container_name}:/service.py')
+ # Validation commands fail on demand without involving real nginx
+ # certificates or Fluent Bit packages; service handlers are real.
+ for binary in ['/usr/sbin/nginx', '/opt/fluent-bit/bin/fluent-bit']:
+ container('sh', '-c', 'cat > "$1"; chmod 755 "$1"', 'sh', binary,
+ input='#!/bin/sh\ntest ! -e /tmp/reject-configuration\n')
+ for name, config in [
+ ('nginx', '/etc/nginx/nginx.conf'),
+ ('node-exporter-proxy', '/etc/taler-monitoring/nginx.conf'),
+ ('prometheus-node-exporter', '/etc/default/prometheus-node-exporter'),
+ ('fluent-bit', '/etc/fluent-bit/fluent-bit.conf'),
+ ('taler-exchange-httpd', '/etc/nginx/nginx.conf')]:
+ container('tee', config, input='1\n')
+ container('tee', f'/etc/systemd/system/{name}.service', input=f'''
+[Unit]
+Description=Disposable deployment recovery fixture
+[Service]
+ExecStart=/usr/bin/python3 /service.py {name} {config}
+[Install]
+WantedBy=multi-user.target
+''')
+ container('systemctl', 'daemon-reload')
+ before = pids()
+ deploy(check=True)
+ assert not marker() and pids() == before
+ deploy()
+ assert not marker() and app_active() and not listener()
+ before = pids()
+ deploy(revision=2, failure_stage='before-handlers', expect_failure=True)
+ assert marker() and not app_active() and pids() == before and not listener()
+ config_time = container('stat', '-c', '%Y', '/etc/fluent-bit/fluent-bit.conf').stdout
+ deploy(revision=2, check=True)
+ assert marker() and not app_active() and pids() == before and not listener()
+ deploy(revision=2, failure_stage='before-handlers', expect_failure=True)
+ assert marker() and not app_active() and pids() == before
+ deploy(revision=2)
+ assert not marker() and app_active() and pids() != before and listener()
+ assert all(pids()[unit] != before[unit] for unit in INFRASTRUCTURE)
+ assert container('stat', '-c', '%Y', '/etc/fluent-bit/fluent-bit.conf').stdout == config_time
+ print('PASS: failed configuration and unchanged retries recover; check mode preserves state', flush=True)
+
+ before = pids()
+ deploy(revision=2)
+ deploy(revision=2, check=True)
+ assert not marker() and app_active() and pids() == before and listener()
+ print('PASS: successful reruns and check mode leave infrastructure running', flush=True)
+
+ container('systemctl', 'stop', 'fluent-bit')
+ container('touch', '/tmp/disable-metrics')
+ deploy(revision=2, expect_failure=True)
+ assert marker() and not app_active() and not listener()
+ container('systemctl', 'is-active', '--quiet', 'fluent-bit')
+ container('rm', '/tmp/disable-metrics')
+ deploy(revision=2)
+ assert not marker() and app_active() and listener()
+ print('PASS: a running collector without a metrics listener fails deployment and recovers on retry', flush=True)
+
+ before = pids()
+ deploy(revision=2, failure_stage='sanity-checks', expect_failure=True)
+ assert marker() and not app_active()
+ container('touch', '/tmp/reject-configuration')
+ deploy(revision=2, expect_failure=True)
+ assert marker() and not app_active() and pids() == before
+ container('rm', '/tmp/reject-configuration')
+ deploy(revision=2, enabled=False)
+ after = pids()
+ assert not marker() and app_active()
+ assert all(after[unit] == before[unit] for unit in INFRASTRUCTURE if unit != 'nginx'), \
+ 'Disabled monitoring was restarted'
+ assert after != before, 'Nginx recovery was skipped with monitoring disabled'
+ print('PASS: late failures and validation failures retain recovery state; disabled monitoring is untouched', flush=True)
+ finally:
+ run('podman', 'rm', '--force', container_name)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/playbooks/setup.yml b/playbooks/setup.yml
@@ -77,6 +77,10 @@
tasks:
- name: Deploy with applications stopped until configuration is complete
block:
+ - name: Track deployment completion before modifying services
+ ansible.builtin.include_role:
+ name: deployment_state
+
- name: Stop existing applications before any package upgrades
ansible.builtin.include_role:
name: stop_services
@@ -146,6 +150,11 @@
ansible.builtin.include_role:
name: post_deployment_checks
+ - name: Record successful deployment
+ ansible.builtin.include_role:
+ name: deployment_state
+ tasks_from: complete
+
rescue:
- name: Leave applications stopped after a deployment failure
ansible.builtin.include_role:
diff --git a/roles/deployment_state/defaults/main.yml b/roles/deployment_state/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+deployment_state_directory: /var/lib/taler-deployment
diff --git a/roles/deployment_state/tasks/complete.yml b/roles/deployment_state/tasks/complete.yml
@@ -0,0 +1,7 @@
+---
+- name: Record successful deployment after handlers and sanity checks
+ ansible.builtin.file:
+ path: "{{ deployment_state_directory }}/incomplete"
+ state: absent
+ changed_when: false
+ when: not ansible_check_mode
diff --git a/roles/deployment_state/tasks/main.yml b/roles/deployment_state/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+- name: Check for an incomplete deployment
+ ansible.builtin.stat:
+ path: "{{ deployment_state_directory }}/incomplete"
+ register: deployment_state_previous
+
+- name: Remember whether configuration handlers need to be recovered
+ ansible.builtin.set_fact:
+ deployment_recover_configuration: "{{ deployment_state_previous.stat.exists }}"
+
+# This bookkeeping is created and removed on every successful deployment.
+# It should not count as a configuration change or modify hosts in check mode.
+- name: Create the private deployment state directory
+ ansible.builtin.file:
+ path: "{{ deployment_state_directory }}"
+ state: directory
+ owner: root
+ group: root
+ mode: "0700"
+ changed_when: false
+ when: not ansible_check_mode
+
+- name: Record the incomplete deployment before modifying services
+ ansible.builtin.copy:
+ content: "Configuration may not have been applied by a completed deployment.\n"
+ dest: "{{ deployment_state_directory }}/incomplete"
+ force: false
+ owner: root
+ group: root
+ mode: "0600"
+ changed_when: false
+ when: not ansible_check_mode
diff --git a/roles/monitoring/handlers/main.yml b/roles/monitoring/handlers/main.yml
@@ -40,9 +40,16 @@
changed_when: true
when: not ansible_check_mode
-- name: Restart monitoring Fluent Bit
+- name: Validate monitoring Fluent Bit before restart
+ ansible.builtin.command: /opt/fluent-bit/bin/fluent-bit --dry-run -c /etc/fluent-bit/fluent-bit.conf
+ changed_when: false
+ listen: Restart monitoring Fluent Bit
+ when: not ansible_check_mode
+
+- name: Restart validated monitoring Fluent Bit
ansible.builtin.systemd_service:
name: fluent-bit
daemon_reload: true
state: restarted
+ listen: Restart monitoring Fluent Bit
when: not ansible_check_mode
diff --git a/roles/monitoring/tasks/main.yml b/roles/monitoring/tasks/main.yml
@@ -103,6 +103,9 @@
daemon_reload: true
when: not ansible_check_mode
+- name: Recover monitoring handlers lost by an interrupted deployment
+ ansible.builtin.import_tasks: recover.yml
+
# Move an existing legacy exporter off port 9100 before starting the proxy.
# This also loads renewed certificate pairs before reporting success.
- name: Apply monitoring configuration before enabling services
@@ -118,3 +121,17 @@
- node-exporter-proxy
- fluent-bit
when: not ansible_check_mode or (item ~ '.service') in ansible_facts['services']
+
+- name: Wait for the enabled exchange latency exporter
+ ansible.builtin.uri:
+ url: http://127.0.0.1:2021/metrics
+ status_code: 200
+ use_proxy: false
+ timeout: 5
+ register: monitoring_latency_exporter_check
+ until: monitoring_latency_exporter_check.status | default(-1) == 200
+ retries: 10
+ delay: 1
+ when:
+ - monitoring_nginx_latency_enabled
+ - not ansible_check_mode
diff --git a/roles/monitoring/tasks/recover.yml b/roles/monitoring/tasks/recover.yml
@@ -0,0 +1,13 @@
+---
+# An interrupted play loses its handler notifications even though its file
+# changes survive. Requeue them only after this role has configured everything.
+- name: Reapply monitoring configuration after an incomplete deployment
+ ansible.builtin.debug:
+ msg: Reapplying monitoring configuration left by an incomplete deployment.
+ changed_when: true
+ when: deployment_recover_configuration | default(false) | bool
+ notify:
+ - Restart monitoring node_exporter
+ - Restart monitoring proxy
+ - Restart monitoring journal
+ - Restart monitoring Fluent Bit
diff --git a/roles/webserver/tasks/main.yml b/roles/webserver/tasks/main.yml
@@ -66,7 +66,9 @@
- name: Validate nginx configuration without removing enabled sites
ansible.builtin.command: nginx -c /etc/nginx/nginx.conf -t
- changed_when: false
+ # Files written by an interrupted deployment no longer notify on a retry.
+ changed_when: deployment_recover_configuration | default(false) | bool
+ notify: Restart nginx
check_mode: false
- name: Ensure Nginx service is enabled and started
diff --git a/test b/test
@@ -29,6 +29,9 @@ python3 contrib/tests/test_monitoring_listeners.py
# Build our image
podman build -f Containerfile -t "$test_image"
+# Exercise interrupted deployment recovery with isolated service fixtures.
+python3 contrib/tests/test_deployment_recovery.py --image "localhost/$test_image"
+
# Run in background (-d) with systemd init. Taler's hardened systemd units
# require capabilities that Podman otherwise removes from the container.
podman run \
