faults.py (17965B)
1 # This file is part of GNU Taler 2 # (C) 2026 Taler Systems S.A. 3 # 4 # GNU Taler is free software; you can redistribute it and/or modify it under the 5 # terms of the GNU General Public License as published by the Free Software 6 # Foundation; either version 3, or (at your option) any later version. 7 # 8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY 9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 10 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License along with 13 # GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 14 """Breaking a backend that works, with `taler-twister`. 15 16 The mock backend answers the ways a *misconfigured* merchant answers: a fresh 17 instance, no bank account, the wrong currency. What it cannot do is be the 18 real backend and go wrong halfway through, because everything it says is 19 something the harness wrote. A reply that is the demo backend's own, with one 20 field deleted or one status code changed, is a different kind of input: the 21 terminal's own parsers and error branches see something they have no reason to 22 expect, and the rest of the exchange is real. 23 24 That is what twister is for -- the fault-injecting proxy GNU Taler's C test 25 suites use. It sits in front of a real backend and, on command, mutates the 26 *next* request or response: a status code, a deleted JSON field, a replaced 27 value, a flipped character, a truncated body. Each mutation is consumed when 28 it fires, which suits a test script issuing one request at a time and does not 29 suit a UI that polls; `Twister.sustain()` covers that by re-arming in a loop. 30 31 Two things have to be bridged to put it under the terminal. 32 33 * **TLS.** The terminal hard-codes `https://` for the merchant URL -- see 34 `ConfigFragment.sanitizeMerchantUrlAndUpdateFields` -- and twister is a plain 35 HTTP proxy. `TlsFront` terminates TLS with the same certificate the mock 36 uses, which the `sim` build type trusts, and relays bytes to twister. A byte 37 relay is enough: nothing here needs to understand the HTTP it carries, so 38 keep-alive, chunking and uploads pass through untouched. 39 * **The path prefix.** Twister appends the request's path to 40 `DESTINATION_BASE_URL`, so the instance path has to be in the terminal's 41 merchant URL and not in the destination. `Twister.merchant_url` says what 42 to configure the terminal with. 43 44 `TlsFront` also injects the faults twister has no notion of, because they 45 happen below HTTP: a connection that closes in the middle of a response body, 46 or one that is reset while the request is still going up. Those are the inputs 47 that reach `Response.getFailureString`'s `IOException` arm, which no status 48 code can produce. 49 """ 50 51 from __future__ import annotations 52 53 import os 54 import re 55 import select 56 import socket 57 import ssl 58 import subprocess 59 import threading 60 import time 61 from contextlib import contextmanager 62 from dataclasses import dataclass, field 63 from pathlib import Path 64 from typing import Iterator, Optional 65 66 HERE = Path(__file__).resolve().parent 67 TLS_DIR = HERE.parent / "tls" 68 69 # Built on the emulator host against its system GNUnet; see README. 70 TWISTER_PREFIX = Path(os.environ.get("POS_TWISTER_PREFIX", "/tmp/twister-build/inst")) 71 SERVICE = os.environ.get("POS_TWISTER_SERVICE", str(TWISTER_PREFIX / "bin" / "taler-twister-service")) 72 CLI = os.environ.get("POS_TWISTER_CLI", str(TWISTER_PREFIX / "bin" / "taler-twister")) 73 74 75 class FaultError(RuntimeError): 76 """The fault could not be armed -- a harness problem, not the app's.""" 77 78 79 def available() -> bool: 80 return Path(SERVICE).is_file() and Path(CLI).is_file() 81 82 83 def _env() -> dict: 84 env = dict(os.environ) 85 lib = str(TWISTER_PREFIX / "lib") 86 env["LD_LIBRARY_PATH"] = lib + ":" + env.get("LD_LIBRARY_PATH", "") 87 return env 88 89 90 @dataclass 91 class Twister: 92 """A `taler-twister-service` in front of one backend. 93 94 `destination` is the backend's origin with no path and no trailing slash 95 (`https://backend.demo.taler.net`); twister concatenates it with whatever 96 path the terminal asks for. `chaos` is a percentage: every proxied request 97 has that chance of coming back 503 without being asked, which is what the 98 random walk wants and what a deliberate scenario does not. 99 """ 100 101 destination: str 102 port: int 103 front_port: int 104 run_dir: Path 105 chaos: int = 0 106 _proc: Optional[subprocess.Popen] = None 107 _front: Optional["TlsFront"] = None 108 _conf: Optional[Path] = None 109 _armed: list[str] = field(default_factory=list) 110 111 @property 112 def host(self) -> str: 113 """What the terminal should be pointed at.""" 114 return f"localhost:{self.front_port}" 115 116 def merchant_url(self, instance: str) -> str: 117 return f"https://{self.host}/instances/{instance}" 118 119 # -- lifecycle -------------------------------------------------------- 120 121 def start(self) -> "Twister": 122 self.run_dir.mkdir(parents=True, exist_ok=True) 123 self._conf = self.run_dir / f"twister-{self.port}.conf" 124 sock = self.run_dir / f"twister-{self.port}.sock" 125 # UNIX_MATCH_UID/GID off: the driver and the service are the same user 126 # here, but a harness run under sudo is not, and the failure that 127 # produces is an unexplained hang in the CLI rather than a refusal. 128 self._conf.write_text( 129 "[twister]\n" 130 f"HTTP_PORT = {self.port}\n" 131 "SERVE = tcp\n" 132 f"DESTINATION_BASE_URL = {self.destination.rstrip('/')}\n" 133 "HOSTNAME = localhost\n" 134 "ACCEPT_FROM = 127.0.0.1;\n" 135 "ACCEPT_FROM6 = ::1;\n" 136 f"UNIXPATH = {sock}\n" 137 "UNIX_MATCH_UID = NO\n" 138 "UNIX_MATCH_GID = NO\n" 139 + (f"CHAOS_RATE = {self.chaos}\n" if self.chaos else "") 140 ) 141 log = open(self.run_dir / f"twister-{self.port}.log", "ab") 142 self._proc = subprocess.Popen( 143 [SERVICE, "-c", str(self._conf)], env=_env(), 144 stdout=log, stderr=subprocess.STDOUT) 145 self._wait_alive() 146 self._front = TlsFront(listen=self.front_port, upstream=self.port).start() 147 return self 148 149 def _wait_alive(self, timeout: float = 15.0) -> None: 150 """Wait for the control socket, which is the last thing to come up.""" 151 deadline = time.time() + timeout 152 last = "" 153 while time.time() < deadline: 154 if self._proc and self._proc.poll() is not None: 155 raise FaultError( 156 f"twister-service exited with {self._proc.returncode}; see " 157 f"{self.run_dir}/twister-{self.port}.log") 158 try: 159 self._cli("-a") 160 return 161 except FaultError as e: 162 last = str(e) 163 time.sleep(0.3) 164 raise FaultError(f"twister-service did not come up: {last}") 165 166 def stop(self) -> None: 167 if self._front is not None: 168 self._front.stop() 169 self._front = None 170 if self._proc is not None: 171 self._proc.terminate() 172 try: 173 self._proc.wait(timeout=5) 174 except subprocess.TimeoutExpired: 175 self._proc.kill() 176 self._proc = None 177 178 def __enter__(self) -> "Twister": 179 return self.start() 180 181 def __exit__(self, *exc) -> None: 182 self.stop() 183 184 # -- arming ----------------------------------------------------------- 185 186 def _cli(self, *args: str) -> str: 187 assert self._conf is not None 188 cmd = [CLI, "-c", str(self._conf), *args] 189 try: 190 out = subprocess.run(cmd, env=_env(), capture_output=True, 191 text=True, timeout=20) 192 except subprocess.TimeoutExpired: 193 raise FaultError(f"twister CLI hung: {' '.join(args)}") 194 if out.returncode != 0: 195 raise FaultError( 196 f"twister CLI failed ({out.returncode}): {' '.join(args)}: " 197 f"{(out.stderr or out.stdout).strip()[:200]}") 198 return out.stdout 199 200 def _arm(self, *args: str, label: str) -> None: 201 self._cli(*args) 202 self._armed.append(label) 203 204 @property 205 def armed(self) -> list[str]: 206 """What has been armed, for the episode's notes.""" 207 return list(self._armed) 208 209 def status(self, code: int) -> None: 210 """Answer the next proxied request with this status code.""" 211 self._arm("-r", str(code), label=f"status {code}") 212 213 def delete(self, path: str) -> None: 214 """Drop a field from the next response (`a.b.0` indexes arrays).""" 215 self._arm("-d", path, label=f"delete {path}") 216 217 def modify(self, path: str, value: str) -> None: 218 """Replace a field in the next response. 219 220 A value that parses as JSON is inserted as JSON, so `"null"` and `"[]"` 221 are objects and anything else is a string. The field has to exist 222 already -- twister replaces, it does not add. 223 """ 224 self._arm("-m", path, "-V", value, label=f"modify {path}={value}") 225 226 def modify_upload(self, path: str, value: str) -> None: 227 """Replace a field in the next *request* the terminal sends.""" 228 self._arm("-X", path, "-V", value, label=f"modify-ul {path}={value}") 229 230 def header(self, name: str, value: str) -> None: 231 """Replace a response header.""" 232 self._arm("-H", name, "-V", value, label=f"header {name}={value}") 233 234 def flip(self, path: str) -> None: 235 """Flip a random character in a string field of the next response.""" 236 self._arm("-f", path, label=f"flip {path}") 237 238 def malform(self) -> None: 239 """Truncate the next response body, so it is not JSON any more.""" 240 self._arm("-M", label="malform") 241 242 def malform_upload(self) -> None: 243 self._arm("-U", label="malform-ul") 244 245 # -- a mutation that outlives one request ----------------------------- 246 247 @contextmanager 248 def sustain(self, *arm: tuple, every: float = 0.4) -> Iterator[None]: 249 """Keep a mutation armed while the block runs. 250 251 A twister mutation fires once. The terminal polls -- the payment 252 screen re-checks the order every second or so -- so a mutation armed 253 once is visible for a single poll and then undone by the next, and the 254 dialog it was supposed to raise opens and closes before the harness can 255 see it. Re-arming in a loop keeps the backend wrong for as long as the 256 scenario needs it to be. 257 258 with tw.sustain(("modify", "taler_pay_uri", insecure)): 259 app.read_dialog() 260 261 Each tuple is a method name and its arguments. Failures inside the 262 loop are collected rather than raised: the loop runs in its own thread, 263 where an exception would be lost, and arming can legitimately fail at 264 the end of a block when the service is already going away. 265 """ 266 stop = threading.Event() 267 errors: list[str] = [] 268 269 def pump() -> None: 270 while not stop.is_set(): 271 for spec in arm: 272 try: 273 getattr(self, spec[0])(*spec[1:]) 274 except FaultError as e: 275 errors.append(str(e)) 276 stop.set() 277 return 278 stop.wait(every) 279 280 for spec in arm: # fail loudly on a bad spec 281 getattr(self, spec[0])(*spec[1:]) 282 thread = threading.Thread(target=pump, daemon=True) 283 thread.start() 284 try: 285 yield 286 finally: 287 stop.set() 288 thread.join(timeout=5) 289 if errors: 290 raise FaultError(errors[0]) 291 292 293 # -------------------------------------------------------------------------- 294 # The TLS front 295 296 297 @dataclass 298 class TlsFront: 299 """TLS termination in front of twister, with transport-level faults. 300 301 `fault` is set between requests and applies to every connection until it 302 is cleared: 303 304 none relay faithfully 305 cut forward the response headers, then close the connection 306 in the middle of the body 307 reset reset the connection as soon as the request is complete 308 trickle relay the response a few bytes at a time 309 310 `cut` is the one that matters. Every HTTP error the terminal can be shown 311 through a status code it already has a branch for; a body that stops 312 arriving halfway is an `IOException` out of the Ktor engine, which is a 313 different arm of `Response.getFailureString` and reaches the terminal as a 314 message nobody wrote. 315 """ 316 317 listen: int 318 upstream: int 319 fault: str = "none" 320 cut_after: int = 200 321 _sock: Optional[socket.socket] = None 322 _ctx: Optional[ssl.SSLContext] = None 323 _stop: threading.Event = field(default_factory=threading.Event) 324 _thread: Optional[threading.Thread] = None 325 seen: int = 0 326 327 def start(self) -> "TlsFront": 328 self._ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) 329 self._ctx.load_cert_chain(certfile=str(TLS_DIR / "srv.crt"), 330 keyfile=str(TLS_DIR / "srv.key")) 331 self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 332 self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 333 self._sock.bind(("127.0.0.1", self.listen)) 334 self._sock.listen(32) 335 self._sock.settimeout(0.5) 336 self._thread = threading.Thread(target=self._accept_loop, daemon=True) 337 self._thread.start() 338 return self 339 340 def stop(self) -> None: 341 self._stop.set() 342 if self._thread is not None: 343 self._thread.join(timeout=3) 344 if self._sock is not None: 345 self._sock.close() 346 self._sock = None 347 348 def _accept_loop(self) -> None: 349 assert self._sock is not None and self._ctx is not None 350 while not self._stop.is_set(): 351 try: 352 raw, _ = self._sock.accept() 353 except socket.timeout: 354 continue 355 except OSError: 356 return 357 self.seen += 1 358 threading.Thread(target=self._serve, args=(raw,), daemon=True).start() 359 360 def _serve(self, raw: socket.socket) -> None: 361 assert self._ctx is not None 362 try: 363 client = self._ctx.wrap_socket(raw, server_side=True) 364 except (ssl.SSLError, OSError): 365 raw.close() 366 return 367 fault = self.fault 368 if fault == "reset": 369 # SO_LINGER with a zero timeout turns close() into an RST, which is 370 # what a peer going away without a FIN looks like. 371 client.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, 372 b"\x01\x00\x00\x00\x00\x00\x00\x00") 373 client.close() 374 return 375 try: 376 up = socket.create_connection(("127.0.0.1", self.upstream), timeout=10) 377 except OSError: 378 client.close() 379 return 380 self._relay(client, up, fault) 381 382 def _relay(self, client: ssl.SSLSocket, up: socket.socket, fault: str) -> None: 383 """Pump both directions until either side is done.""" 384 sent_down = 0 385 headers_done = False 386 try: 387 client.setblocking(False) 388 up.setblocking(False) 389 while True: 390 ready, _, _ = select.select([client, up], [], [], 30) 391 if not ready: 392 return 393 for sock in ready: 394 try: 395 data = sock.recv(65536) 396 except ssl.SSLWantReadError: 397 continue 398 except OSError: 399 return 400 if not data: 401 return 402 if sock is client: 403 up.sendall(data) 404 continue 405 if fault == "trickle": 406 for i in range(0, len(data), 8): 407 client.sendall(data[i:i + 8]) 408 time.sleep(0.05) 409 continue 410 if fault == "cut": 411 if not headers_done and b"\r\n\r\n" in data: 412 head, _, body = data.partition(b"\r\n\r\n") 413 client.sendall(head + b"\r\n\r\n") 414 headers_done = True 415 data = body 416 if headers_done: 417 room = max(0, self.cut_after - sent_down) 418 if room: 419 client.sendall(data[:room]) 420 sent_down += min(room, len(data)) 421 if sent_down >= self.cut_after: 422 return 423 continue 424 client.sendall(data) 425 finally: 426 for sock in (client, up): 427 try: 428 sock.close() 429 except OSError: 430 pass 431 432 @contextmanager 433 def transport(self, fault: str) -> Iterator[None]: 434 """Apply a transport fault for the duration of the block.""" 435 before = self.fault 436 self.fault = fault 437 try: 438 yield 439 finally: 440 self.fault = before 441 442 443 def front_of(tw: Twister) -> TlsFront: 444 assert tw._front is not None, "twister not started" 445 return tw._front 446 447 448 # The pay URI the terminal is supposed to object to. `taler+http://` is what a 449 # backend behind a plain-HTTP reverse proxy hands out, and the terminal offers 450 # to upgrade it, to use it anyway, or to cancel the order -- three branches of 451 # `PaymentManager` that no correctly configured backend can reach. 452 def insecure_pay_uri(uri: str) -> str: 453 return re.sub(r"^taler://", "taler+http://", uri)