test_deployment_recovery.py (12337B)
1 #!/usr/bin/env python3 2 """Exercise setup's failure/retry flow with disposable systemd service fixtures. 3 4 Uses the real setup play, deployment state, recovery tasks and service handlers. 5 Package installation and application configuration are replaced by small roles 6 so the regression needs no Taler deployment, certificates or package downloads. 7 """ 8 import argparse 9 import json 10 import os 11 from pathlib import Path 12 import shutil 13 import subprocess 14 import tempfile 15 import time 16 17 import yaml 18 19 REPO = Path(__file__).resolve().parents[2] 20 INFRASTRUCTURE = ['nginx', 'prometheus-node-exporter', 'node-exporter-proxy', 'fluent-bit'] 21 22 23 def run(*args, **kwargs): 24 return subprocess.run(list(map(str, args)), capture_output=True, text=True, 25 check=kwargs.pop('check', True), **kwargs) 26 27 28 def main(): 29 parser = argparse.ArgumentParser(description=__doc__) 30 parser.add_argument('--image', default='localhost/ansible-taler-test:latest', 31 help='Local Debian image with systemd, Python and dbus installed') 32 args = parser.parse_args() 33 with tempfile.TemporaryDirectory(prefix='taler-deployment-recovery-') as directory: 34 work = Path(directory) 35 container_name = 'taler-deployment-recovery-' + str(os.getpid()) 36 37 def container(*cmd, **kwargs): 38 return run('podman', 'exec', '-i', container_name, *cmd, **kwargs) 39 40 def write(path, value): 41 path.parent.mkdir(parents=True, exist_ok=True) 42 path.write_text(yaml.safe_dump(value, sort_keys=False)) 43 44 def role(name, tasks): 45 write(work / 'roles' / name / 'tasks/main.yml', tasks) 46 47 def production_tasks(role_name): 48 return yaml.safe_load((REPO / 'roles' / role_name / 'tasks/main.yml').read_text()) 49 50 def fail(stage): 51 return {'name': 'Inject deployment failure at ' + stage, 52 'ansible.builtin.fail': {'msg': 'Injected regression failure'}, 53 'when': f"failure_stage == '{stage}'"} 54 55 shutil.copyfile(REPO / 'playbooks/setup.yml', work / 'setup.yml') 56 shutil.copytree(REPO / 'roles/deployment_state', work / 'roles/deployment_state') 57 for name in ['common_packages', 'database', 'libeufin-nexus', 'challenger', 58 'exchange', 'auditor', 'devtesting']: 59 role(name, []) 60 role('stop_services', [ 61 {'ansible.builtin.systemd_service': {'name': 'taler-exchange-httpd', 'state': 'stopped'}}, 62 {'ansible.builtin.set_fact': {'stop_services_active_units': []}}]) 63 role('start_services', [ 64 {'ansible.builtin.systemd_service': {'name': 'taler-exchange-httpd', 'state': 'started'}}]) 65 role('post_deployment_checks', [fail('sanity-checks')]) 66 nginx_tasks = production_tasks('webserver') 67 role('webserver', [ 68 {'ansible.builtin.copy': {'content': '{{ revision }}\n', 69 'dest': '/etc/nginx/nginx.conf', 'mode': '0644'}, 70 'notify': 'Restart nginx'}, 71 *nginx_tasks[-2:]]) # Real validation/recovery notification and enablement. 72 for name in ['webserver', 'monitoring']: 73 shutil.copytree(REPO / 'roles' / name / 'handlers', work / 'roles' / name / 'handlers') 74 monitor_tasks = production_tasks('monitoring') 75 apply_index = next(i for i, task in enumerate(monitor_tasks) 76 if task['name'] == 'Load the configured monitoring units') 77 role('monitoring', [ 78 monitor_tasks[1], # Service discovery for check mode. 79 {'ansible.builtin.copy': {'content': '{{ revision }}\n', 80 'dest': '{{ item }}', 'mode': '0600'}, 81 'loop': ['/etc/fluent-bit/fluent-bit.conf', '/etc/taler-monitoring/nginx.conf', 82 '/etc/default/prometheus-node-exporter'], 83 'notify': ['Restart monitoring Fluent Bit', 'Restart monitoring proxy', 84 'Restart monitoring node_exporter', 'Restart monitoring journal']}, 85 fail('before-handlers'), *monitor_tasks[apply_index:]]) 86 shutil.copyfile(REPO / 'roles/monitoring/tasks/recover.yml', 87 work / 'roles/monitoring/tasks/recover.yml') 88 write(work / 'roles/monitoring/tasks/preflight.yml', []) 89 write(work / 'inventory.yml', {'all': {'hosts': {container_name: { 90 'ansible_connection': 'containers.podman.podman', 'ansible_user': 'root', 91 'ansible_python_interpreter': '/usr/bin/python3'}}}}) 92 (work / 'ansible.cfg').write_text('[defaults]\nroles_path = ' + str(work / 'roles') + '\n') 93 environment = dict(os.environ, ANSIBLE_CONFIG=str(work / 'ansible.cfg'), ANSIBLE_NOCOWS='1') 94 95 def deploy(revision=1, failure_stage='', check=False, enabled=True, expect_failure=False): 96 variables = dict(revision=revision, failure_stage=failure_stage, 97 enable_monitoring=enabled, monitoring_nginx_latency_enabled=revision >= 2, 98 deployment_kind='regression', exchange_attribute_encryption_key='fixture', 99 devtesting_mock_mfa=False, deploy_auditor=False, deploy_challenger=False, 100 use_ebics=False, configure_ebics=False) 101 write(work / 'vars.yml', variables) 102 command = ['ansible-playbook', '-i', work / 'inventory.yml', work / 'setup.yml', 103 '-e', '@' + str(work / 'vars.yml')] 104 if check: 105 command.append('--check') 106 result = run(*command, env=environment, check=False) 107 if bool(result.returncode) != expect_failure: 108 raise AssertionError(result.stdout + result.stderr) 109 return result.stdout 110 111 def pids(): 112 values = container('systemctl', 'show', '--value', '-p', 'MainPID', *INFRASTRUCTURE).stdout.split() 113 return dict(zip(INFRASTRUCTURE, values, strict=True)) 114 115 def marker(): 116 return container('test', '-f', '/var/lib/taler-deployment/incomplete', check=False).returncode == 0 117 118 def app_active(): 119 return container('systemctl', 'is-active', '--quiet', 'taler-exchange-httpd', check=False).returncode == 0 120 121 def listener(): 122 return container('python3', '-c', 123 'import urllib.request; print(urllib.request.urlopen(' 124 '"http://127.0.0.1:2021/metrics", timeout=2).status)', check=False).returncode == 0 125 126 run('podman', 'run', '--detach', '--name', container_name, '--network', 'none', 127 '--systemd', 'always', '--tmpfs', '/run', '--tmpfs', '/tmp', 128 args.image, '/sbin/init') 129 try: 130 for _ in range(30): 131 state = container('systemctl', 'is-system-running', check=False).stdout.strip() 132 if state in ['running', 'degraded']: 133 break 134 time.sleep(1) 135 else: 136 raise AssertionError('Test systemd did not start') 137 # Each fixture reads its configuration at startup. Fluent Bit's 138 # second revision opens the latency listener, just as on rusty. 139 service = work / 'service.py' 140 service.write_text('''#!/usr/bin/python3 141 import http.server, pathlib, sys, time 142 name, config = sys.argv[1:] 143 revision = pathlib.Path(config).read_text().strip() 144 if name == 'fluent-bit' and int(revision) >= 2 and not pathlib.Path('/tmp/disable-metrics').exists(): 145 class Metrics(http.server.BaseHTTPRequestHandler): 146 def do_GET(self): 147 self.send_response(200) 148 self.end_headers() 149 def log_message(self, *args): 150 pass 151 http.server.HTTPServer(('127.0.0.1', 2021), Metrics).serve_forever() 152 else: 153 time.sleep(3600) 154 ''') 155 container('mkdir', '-p', '/etc/nginx', '/etc/taler-monitoring', '/etc/fluent-bit', 156 '/opt/fluent-bit/bin') 157 run('podman', 'cp', service, f'{container_name}:/service.py') 158 # Validation commands fail on demand without involving real nginx 159 # certificates or Fluent Bit packages; service handlers are real. 160 for binary in ['/usr/sbin/nginx', '/opt/fluent-bit/bin/fluent-bit']: 161 container('sh', '-c', 'cat > "$1"; chmod 755 "$1"', 'sh', binary, 162 input='#!/bin/sh\ntest ! -e /tmp/reject-configuration\n') 163 for name, config in [ 164 ('nginx', '/etc/nginx/nginx.conf'), 165 ('node-exporter-proxy', '/etc/taler-monitoring/nginx.conf'), 166 ('prometheus-node-exporter', '/etc/default/prometheus-node-exporter'), 167 ('fluent-bit', '/etc/fluent-bit/fluent-bit.conf'), 168 ('taler-exchange-httpd', '/etc/nginx/nginx.conf')]: 169 container('tee', config, input='1\n') 170 container('tee', f'/etc/systemd/system/{name}.service', input=f''' 171 [Unit] 172 Description=Disposable deployment recovery fixture 173 [Service] 174 ExecStart=/usr/bin/python3 /service.py {name} {config} 175 [Install] 176 WantedBy=multi-user.target 177 ''') 178 container('systemctl', 'daemon-reload') 179 before = pids() 180 deploy(check=True) 181 assert not marker() and pids() == before 182 deploy() 183 assert not marker() and app_active() and not listener() 184 before = pids() 185 deploy(revision=2, failure_stage='before-handlers', expect_failure=True) 186 assert marker() and not app_active() and pids() == before and not listener() 187 config_time = container('stat', '-c', '%Y', '/etc/fluent-bit/fluent-bit.conf').stdout 188 deploy(revision=2, check=True) 189 assert marker() and not app_active() and pids() == before and not listener() 190 deploy(revision=2, failure_stage='before-handlers', expect_failure=True) 191 assert marker() and not app_active() and pids() == before 192 deploy(revision=2) 193 assert not marker() and app_active() and pids() != before and listener() 194 assert all(pids()[unit] != before[unit] for unit in INFRASTRUCTURE) 195 assert container('stat', '-c', '%Y', '/etc/fluent-bit/fluent-bit.conf').stdout == config_time 196 print('PASS: failed configuration and unchanged retries recover; check mode preserves state', flush=True) 197 198 before = pids() 199 deploy(revision=2) 200 deploy(revision=2, check=True) 201 assert not marker() and app_active() and pids() == before and listener() 202 print('PASS: successful reruns and check mode leave infrastructure running', flush=True) 203 204 container('systemctl', 'stop', 'fluent-bit') 205 container('touch', '/tmp/disable-metrics') 206 deploy(revision=2, expect_failure=True) 207 assert marker() and not app_active() and not listener() 208 container('systemctl', 'is-active', '--quiet', 'fluent-bit') 209 container('rm', '/tmp/disable-metrics') 210 deploy(revision=2) 211 assert not marker() and app_active() and listener() 212 print('PASS: a running collector without a metrics listener fails deployment and recovers on retry', flush=True) 213 214 before = pids() 215 deploy(revision=2, failure_stage='sanity-checks', expect_failure=True) 216 assert marker() and not app_active() 217 container('touch', '/tmp/reject-configuration') 218 deploy(revision=2, expect_failure=True) 219 assert marker() and not app_active() and pids() == before 220 container('rm', '/tmp/reject-configuration') 221 deploy(revision=2, enabled=False) 222 after = pids() 223 assert not marker() and app_active() 224 assert all(after[unit] == before[unit] for unit in INFRASTRUCTURE if unit != 'nginx'), \ 225 'Disabled monitoring was restarted' 226 assert after != before, 'Nginx recovery was skipped with monitoring disabled' 227 print('PASS: late failures and validation failures retain recovery state; disabled monitoring is untouched', flush=True) 228 finally: 229 run('podman', 'rm', '--force', container_name) 230 231 232 if __name__ == '__main__': 233 main()