oracles.py (21143B)
1 # This file is part of GNU Taler 2 # (C) 2026 Taler Systems S.A. 3 # 4 # GNU Taler is free software; you can redistribute it and/or modify it under the 5 # terms of the GNU General Public License as published by the Free Software 6 # Foundation; either version 3, or (at your option) any later version. 7 # 8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY 9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 10 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License along with 13 # GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 14 """What counts as a failure. 15 16 An ordinary error is not a finding. A terminal pointed at a backend that is 17 down is supposed to say so; a refund larger than the order is supposed to be 18 refused. A finding is the terminal breaking its own contract: a crash, an ANR, 19 a dialog that names no cause the cashier can act on, an exception message 20 shown as if it were one, a screen that waits forever, a state on screen that 21 the backend contradicts, or a payment code the customer cannot use. 22 23 Two things make this different from the wallet's equivalent. There is no 24 wallet-core here, so there is no error-code registry to lean on -- every 25 judgement comes from the screen, the log and the backend. And the terminal is 26 one half of a transaction, so the backend is an oracle in its own right: what 27 the order really is, as opposed to what the terminal says it is. 28 """ 29 30 from __future__ import annotations 31 32 import hashlib 33 import re 34 from dataclasses import dataclass, field 35 from typing import Iterable, Optional 36 37 from device import PKG, Device, Screen 38 39 CRASH_PATTERNS = ( 40 ("crash", re.compile(r"FATAL EXCEPTION")), 41 ("anr", re.compile(r"ANR in ")), 42 ("native-crash", re.compile(r"\*\*\* \*\*\* \*\*\*|signal \d+ \(SIG")), 43 # The terminal asserts its way through a configuration it believes in: 44 # `configManager.merchantConfig!!` appears in nine places across the 45 # payment, history and refund managers, and a session that ends while one 46 # of them is in flight is a KotlinNullPointerException rather than an 47 # error message. 48 ("kotlin-npe", re.compile(r"KotlinNullPointerException|" 49 r"NullPointerException.*merchantConfig|" 50 r"lateinit property .* has not been initialized")), 51 ("strict-mode", re.compile(r"StrictMode policy violation")), 52 ("main-thread-io", re.compile(r"NetworkOnMainThreadException")), 53 ) 54 55 # Dialog and screen text that no cashier can act on. These are the terminal's 56 # own words, so matching them is exact rather than heuristic. 57 USELESS_MESSAGES = ( 58 ("catch-all-config-error", re.compile(r"^Error: Invalid configuration\.?$")), 59 ("catch-all-malformed", re.compile(r"^Error: The configuration JSON is malformed\.?$")), 60 ) 61 62 # Text that should never reach a user. 63 LEAK_PATTERNS = ( 64 ("placeholder-leak", re.compile(r"%[0-9]+\$[sd]|%s(?![a-zA-Z])|%d(?![a-zA-Z])")), 65 ("null-leak", re.compile(r"(?<![A-Za-z])(null|undefined|NaN|Infinity)(?![A-Za-z])")), 66 ("exception-leak", re.compile(r"java\.lang\.|java\.net\.|javax\.net\.|" 67 r"kotlin\.[A-Za-z]*Exception|" 68 r"[A-Za-z]{3,}Exception(:|\b)|" 69 r"io\.ktor\.|UnknownHostException|" 70 r"SSLHandshakeException|ConnectException")), 71 # A word stuck to a number, like "KUDOS1.00": a currency whose symbol is a 72 # word, formatted as if it were a sign. 73 ("glued-currency", re.compile(r"(?:^|[\s:])[A-Z]{3,}\d+[.,]\d\d(?:$|\s)")), 74 ("raw-resource-name", re.compile(r"\b(?:config|payment|refund|history|order|" 75 r"settings|error|toast|menu)_[a-z_]{3,}\b")), 76 ) 77 # Labels that legitimately contain such words. The About block prints a URL 78 # and a version; a product description is the merchant's own text and may 79 # contain anything, including the word "null" -- but only the sandbox 80 # instance's inventory does, which is why this is narrow. 81 LEAK_EXEMPT = re.compile( 82 r"(?i)^https?://|^payto://|^taler(\+http)?://|^taler-pos://" 83 r"|^[0-9a-f-]{20,}$|^\d{4}\.\d{3}-[A-Z0-9]+$" 84 ) 85 86 # Kinds that describe the terminal behaving reasonably (refusing bad input, 87 # waiting for a counterparty) rather than a defect. They are still recorded: 88 # whether a refusal was right depends on what the scenario was trying to do, 89 # and the write-up needs the evidence either way. 90 # "empty-hierarchy" is here because an empty dump is almost always 91 # `uiautomator` losing a race, not a screen with nothing on it: a real one 92 # shows up as "blank-screen", which has nodes to count. Episode.observe 93 # retries the dump, so one that is still empty is worth keeping -- but not 94 # worth calling a defect. 95 INFORMATIONAL = {"error-dialog", "harness-crash", "slow-screen", 96 "logged-backend-error", "empty-hierarchy"} 97 98 99 @dataclass(frozen=True) 100 class Finding: 101 kind: str 102 detail: str 103 evidence: str = "" 104 105 @property 106 def fingerprint(self) -> str: 107 """Dedup key: the same bug found twice should collapse to one entry.""" 108 norm = re.sub(r"\b\d{4}\.\d{3}-[A-Z0-9]+\b", "<ORDER>", self.detail) 109 norm = re.sub(r"[0-9A-Z]{20,}", "<ID>", norm) 110 norm = re.sub(r"\b\d+\b", "#", norm) 111 norm = re.sub(r"0x[0-9a-f]+", "<ADDR>", norm) 112 return hashlib.sha256(f"{self.kind}|{norm}".encode()).hexdigest()[:16] 113 114 def __str__(self) -> str: 115 return f"[{self.kind}] {self.detail}" 116 117 118 def from_logcat(log: str, pkg: str = PKG) -> list[Finding]: 119 """Crashes, ANRs and the exceptions the terminal swallows. 120 121 A crash is only the app's when the trace says so: `uiautomator dump` dies 122 with "UiAutomationService ... already registered!" whenever two dumps 123 overlap, and that is the driver's problem. 124 """ 125 out: list[Finding] = [] 126 lines = log.splitlines() 127 for kind, pattern in CRASH_PATTERNS: 128 for i, line in enumerate(lines): 129 if not pattern.search(line): 130 continue 131 context = "\n".join(lines[i:i + 14]) 132 if kind in ("crash", "native-crash", "anr") and pkg not in context: 133 out.append(Finding("harness-crash", 134 f"not the app: {line.strip()[:200]}", context)) 135 continue 136 out.append(Finding(kind, line.strip()[:400], context)) 137 return out 138 139 140 # The terminal logs every backend failure and then decides what, if anything, 141 # to show. An exception logged while nothing appeared on screen is the shape 142 # of a swallowed error. 143 SWALLOWED = re.compile( 144 r"(?:ConfigManager|ConfigFragment|PaymentManager|HistoryManager|" 145 r"RefundManager|OrderManager)\s*:.*" 146 r"(Error|Failed|failed|Exception)") 147 148 149 def swallowed_errors(log: str) -> list[Finding]: 150 """Backend failures the terminal logged and did not surface. 151 152 Recorded as information on their own; `silent_failure` is what turns one 153 into a finding, by checking that the screen did not change either. 154 """ 155 out = [] 156 for line in log.splitlines(): 157 if SWALLOWED.search(line): 158 out.append(Finding("logged-backend-error", line.strip()[:300], line)) 159 return out 160 161 162 def dialog_quality(screen: Screen) -> list[Finding]: 163 """Error dialogs judged on whether they say anything. 164 165 The terminal has one dialog for every failure, and what goes in it is 166 decided by a small number of `when` branches -- so the interesting 167 question is not "was there an error" but "did the message name a cause". 168 Three shapes are defects: a catch-all that covers several unrelated 169 failures, a dialog whose message is empty, and one that quotes a stack 170 trace or a Ktor exception at the cashier. 171 """ 172 texts = [t for t in screen.texts() if t] 173 if not texts: 174 return [] 175 has_ok = any(re.fullmatch(r"(?i)ok", t) for t in texts) 176 if not has_ok: 177 return [] 178 body = " | ".join(texts) 179 out: list[Finding] = [] 180 for kind, pattern in USELESS_MESSAGES: 181 for text in texts: 182 if pattern.match(text.strip()): 183 out.append(Finding( 184 kind, 185 f"the only thing the dialog says is {text.strip()!r}, which " 186 f"covers several unrelated failures", 187 body[:800])) 188 # A dialog whose title is the app's name and whose message is the app's 189 # name too has lost its text somewhere: the dialog falls back to the app 190 # name as a title when it has no detail, so both being the app name means 191 # the message never arrived. 192 app_titles = [t for t in texts if t.strip() == "Taler POS"] 193 if len(app_titles) >= 2: 194 out.append(Finding("empty-dialog", 195 "an error dialog with the app name and nothing else", 196 body[:800])) 197 return out 198 199 200 def screen_leaks(screen: Screen) -> list[Finding]: 201 """Rendering that is obviously wrong, whatever the screen is for. 202 203 An unsubstituted format string, a stray "null", a resource name shown as a 204 label, an exception class name or a currency word glued to a number are 205 defects no matter which flow produced them, and they are cheap to check on 206 every dump. 207 """ 208 out: list[Finding] = [] 209 for text in screen.texts(): 210 label = text.strip() 211 if not label or LEAK_EXEMPT.search(label): 212 continue 213 for kind, pattern in LEAK_PATTERNS: 214 if pattern.search(label): 215 out.append(Finding(kind, f"{label[:160]!r} on {screen.activity}", 216 " | ".join(screen.texts())[:800])) 217 break 218 return out 219 220 221 def blank_screen(screen: Screen) -> list[Finding]: 222 """A screen with nothing to read and nothing to tap is a dead end.""" 223 if not screen.nodes: 224 return [Finding("empty-hierarchy", "no view hierarchy returned")] 225 if not screen.labelled() and not screen.tappable(): 226 return [Finding("blank-screen", 227 f"{len(screen.nodes)} nodes, none labelled or tappable")] 228 return [] 229 230 231 def unnamed_controls(screen: Screen) -> list[Finding]: 232 """Controls a screen reader cannot announce. 233 234 Only reported for the terminal's own screens, and only for controls that 235 have no label anywhere inside them: a tile whose text is a child is read 236 correctly by TalkBack, an icon button with no contentDescription is not. 237 """ 238 if not screen.activity.startswith(PKG): 239 return [] 240 anonymous = screen.unnamed_clickables() 241 if not anonymous: 242 return [] 243 return [Finding( 244 "unnamed-control", 245 f"{len(anonymous)} clickable control(s) with no accessible name on " 246 f"{screen.activity}: {[n.center for n in anonymous[:6]]}", 247 "\n".join(str(n) for n in anonymous[:12]), 248 )] 249 250 251 # A spinner that is still spinning after this long is not loading any more. 252 STUCK_AFTER = 75.0 253 254 255 def stuck_progress(dev: Device, screen: Screen, since: float, 256 now: float) -> list[Finding]: 257 """A screen that is still saying "please wait" long after it should be. 258 259 "Fetching configuration..." is the one the terminal can genuinely get 260 stuck on: `ConfigFetcherFragment` shows a spinner and waits for a result 261 that a dropped connection never delivers. 262 """ 263 if not screen.find(r"Fetching configuration|Updating order history"): 264 return [] 265 waited = now - since 266 if waited < STUCK_AFTER: 267 return [] 268 return [Finding( 269 "stuck-progress", 270 f"still showing a progress message after {waited:.0f}s on " 271 f"{screen.activity}", 272 " | ".join(screen.texts())[:500], 273 )] 274 275 276 LOGCAT_STAMP = re.compile(r"^(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)\.(\d+)") 277 TAIL_ERROR = re.compile( 278 r"(?:E|W)/?\s*(?:ConfigManager|PaymentManager|HistoryManager|RefundManager|" 279 r"ConfigFragment|OrderManager)" 280 r"|Error retrieving merchant config" 281 r"|Error (?:fetching|processing|handling)" 282 r"|FATAL EXCEPTION") 283 284 285 def recent_lines(log: str, seconds: float = 6.0) -> list[str]: 286 """Log lines written within the last few seconds. 287 288 logcat has no "since my last read" option, so the timestamps are compared 289 against the wall clock. The device and the host share it closely enough 290 for a window this wide. 291 """ 292 import time 293 now = time.localtime() 294 cutoff = time.time() - seconds 295 out = [] 296 for line in log.splitlines(): 297 m = LOGCAT_STAMP.match(line) 298 if not m: 299 continue 300 month, day, hour, minute, second = (int(m.group(i)) for i in range(1, 6)) 301 stamp = time.mktime((now.tm_year, month, day, hour, minute, second, 302 0, 0, now.tm_isdst)) 303 if stamp >= cutoff: 304 out.append(line) 305 return out 306 307 308 # Labels whose tap is *meant* to change nothing on screen, or which are not 309 # controls at all. Without this the oracle fires on every price in the order 310 # grid during a random walk, because the terminal's managers log a backend 311 # error every few seconds against a busy demo instance and the two get 312 # correlated by nothing more than happening at the same time. 313 NOT_AN_ACTION = re.compile( 314 r"^\s*(?:[\u20ac$\u00a3\u00a5\u20b9\u30af]\s?)?\d+[.,]\d+\s*$" # a price 315 r"|^(?:out of stock|unavailable|wrong currency|unpaid|paid|refunded" 316 r"|payment pending|payment claimed|refund pending)$" 317 r"|^(?:copy|share)$" # clipboard and share-sheet 318 r"|^(?:ref|receipt)\s*#", # an order reference 319 re.I, 320 ) 321 322 323 def silent_failure(dev: Device, label: str, before: str, after: str, 324 lines: int = 300) -> list[Finding]: 325 """A primary action that did nothing, while the log says it failed. 326 327 The terminal's managers log every backend error and then decide what to 328 show; several branches decide to show nothing. Comparing the screen 329 before and after a tap catches the whole class without having to know 330 which branch it was. 331 """ 332 if before != after: 333 return [] 334 if NOT_AN_ACTION.search(label.strip()): 335 return [] 336 activity = dev.dump().activity 337 if activity and not activity.startswith(PKG): 338 return [] 339 tail = dev.adb("logcat", "-d", "-t", str(lines), check=False) 340 fresh = "\n".join(recent_lines(tail)) 341 errors = [ln for ln in fresh.splitlines() if TAIL_ERROR.search(ln)] 342 if not errors: 343 return [] 344 return [Finding( 345 "silent-failure", 346 f"tapping {label!r} changed nothing on screen while the log reported " 347 f"a backend error", 348 "\n".join(errors[-8:])[:2000], 349 )] 350 351 352 # -- the backend as an oracle -------------------------------------------- 353 354 355 def payment_code_usable(screen: Screen, offered: Optional[str], 356 expected: Optional[str]) -> list[Finding]: 357 """The payment screen must offer a code the customer can actually use. 358 359 Three ways it can fail, all of them silent on screen: the terminal shows 360 the "scan this" prompt while having no URI at all; it offers a URI for a 361 different order than the one the backend created; or it offers a 362 `taler+http://` URI, which a wallet with its security defaults will not 363 open -- the terminal has a dialog for exactly that case, so reaching the 364 payment screen without it is a defect rather than a policy choice. 365 """ 366 if not screen.find(r"scan QR Code"): 367 return [] 368 out: list[Finding] = [] 369 if offered is None: 370 out.append(Finding( 371 "payment-without-code", 372 "the payment screen asks the customer to scan a code while the " 373 "terminal is publishing none", 374 " | ".join(screen.texts())[:500])) 375 return out 376 if offered.lower().startswith("taler+http://"): 377 out.append(Finding( 378 "insecure-payment-code", 379 f"the terminal offered an insecure pay URI without asking: " 380 f"{offered[:120]}", 381 offered)) 382 if expected and offered.split("?")[0] != expected.split("?")[0]: 383 out.append(Finding( 384 "payment-code-mismatch", 385 f"the terminal offers {offered[:90]} while the backend's order is " 386 f"{expected[:90]}", 387 f"offered: {offered}\nbackend: {expected}")) 388 return out 389 390 391 PAID_MARKS = re.compile(r"(?i)^(Paid|Refunded|Refund pending)$") 392 UNPAID_MARKS = re.compile(r"(?i)^(Unpaid|Payment pending|Payment claimed)$") 393 394 395 def history_matches_backend(screen: Screen, backend_orders: list[dict] 396 ) -> list[Finding]: 397 """What the history list says about an order, against what the order is. 398 399 Only orders the list actually shows are judged, and only the paid/unpaid 400 distinction -- that is the one a cashier acts on, and the one the terminal 401 derives rather than reads. 402 """ 403 by_id = {o.get("order_id"): o for o in backend_orders} 404 out: list[Finding] = [] 405 labels = [n for n in screen.nodes if n.label.strip()] 406 for node in labels: 407 m = re.match(r"(?i)^(?:Ref|Receipt)\s*#\s*(\S+)", node.label.strip()) 408 if not m: 409 continue 410 order = by_id.get(m.group(1)) 411 if order is None: 412 continue 413 # The status chip sits on the same row as the reference. 414 row = [n.label.strip() for n in labels 415 if abs(n.center[1] - node.center[1]) < 70] 416 shown_paid = any(PAID_MARKS.match(t) for t in row) 417 shown_unpaid = any(UNPAID_MARKS.match(t) for t in row) 418 if not (shown_paid or shown_unpaid): 419 continue 420 really_paid = bool(order.get("paid")) 421 if really_paid and shown_unpaid and not shown_paid: 422 out.append(Finding( 423 "history-state-wrong", 424 f"order {m.group(1)} is paid at the backend and listed as " 425 f"{[t for t in row if UNPAID_MARKS.match(t)][0]!r}", 426 f"row: {row}\nbackend: {order}")) 427 if not really_paid and shown_paid and not shown_unpaid: 428 out.append(Finding( 429 "history-state-wrong", 430 f"order {m.group(1)} is unpaid at the backend and listed as " 431 f"{[t for t in row if PAID_MARKS.match(t)][0]!r}", 432 f"row: {row}\nbackend: {order}")) 433 return out 434 435 436 def orphaned_orders(before: list[dict], after: list[dict], 437 allowed: Iterable[str] = ()) -> list[Finding]: 438 """Unpaid orders the terminal created and left behind. 439 440 Cancelling a payment is supposed to delete the order, and the terminal 441 says so in its log ("Deleting cancelled and unpaid order ..."); an unpaid 442 order that outlives the scenario means that delete did not happen, and the 443 merchant's order list fills up with rows nobody can explain. 444 """ 445 allowed = set(allowed) 446 known = {o.get("order_id") for o in before} 447 leftover = [o for o in after 448 if o.get("order_id") not in known 449 and not o.get("paid") 450 and o.get("order_id") not in allowed] 451 if not leftover: 452 return [] 453 return [Finding( 454 "orphaned-order", 455 f"{len(leftover)} unpaid order(s) left at the backend after the " 456 f"payment was cancelled: {[o.get('order_id') for o in leftover[:5]]}", 457 str(leftover[:5])[:1200], 458 )] 459 460 461 @dataclass 462 class Session: 463 """Collects findings over an episode, deduplicated.""" 464 465 findings: dict[str, Finding] = field(default_factory=dict) 466 467 def add(self, found: Iterable[Finding]) -> list[Finding]: 468 fresh = [] 469 for f in found: 470 if f.fingerprint not in self.findings: 471 self.findings[f.fingerprint] = f 472 fresh.append(f) 473 return fresh 474 475 def check(self, dev: Device, screen: Optional[Screen] = None, 476 log: Optional[str] = None, 477 progress_since: Optional[float] = None) -> list[Finding]: 478 import time 479 found: list[Finding] = [] 480 # Screen oracles only apply to the terminal's own screens. A random 481 # walk leaves the app now and then -- the tutorial link opens a browser 482 # -- and a browser's suggestions are full of the words these oracles 483 # look for. 484 if screen is not None and screen.activity \ 485 and not screen.activity.startswith(PKG): 486 screen = None 487 if screen is not None: 488 found += blank_screen(screen) 489 found += screen_leaks(screen) 490 found += dialog_quality(screen) 491 found += unnamed_controls(screen) 492 if progress_since is not None: 493 found += stuck_progress(dev, screen, progress_since, time.monotonic()) 494 if log is not None: 495 found += from_logcat(log) 496 return self.add(found) 497 498 def serious(self) -> list[Finding]: 499 """Everything except what only describes the terminal refusing or waiting.""" 500 return [f for f in self.findings.values() if f.kind not in INFORMATIONAL]