report.py (5172B)
1 # This file is part of GNU Taler 2 # (C) 2026 Taler Systems S.A. 3 # 4 # GNU Taler is free software; you can redistribute it and/or modify it under the 5 # terms of the GNU General Public License as published by the Free Software 6 # Foundation; either version 3, or (at your option) any later version. 7 # 8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY 9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 10 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License along with 13 # GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 14 """Summarise episode directories and the findings in them. 15 16 python3 report.py the newest run 17 python3 report.py 'suite-2026*' every run matching 18 python3 report.py --notes with what each scenario arranged 19 python3 report.py --kinds findings grouped by kind 20 python3 report.py --unreached only the scenarios that failed to 21 reach their feature 22 """ 23 24 from __future__ import annotations 25 26 import argparse 27 import json 28 import os 29 from collections import Counter, defaultdict 30 from pathlib import Path 31 32 SIM_ROOT = Path(os.environ.get("POS_SIM_ROOT", "/tmp/taler-pos-sim")) 33 OUT = SIM_ROOT / "out" 34 35 # Kinds that only describe the terminal refusing or waiting; see oracles.py. 36 INFORMATIONAL = {"error-dialog", "harness-crash", "slow-screen", 37 "logged-backend-error", "empty-hierarchy"} 38 39 40 def episodes(patterns: list[str]) -> list[Path]: 41 roots: list[Path] = [] 42 if not patterns: 43 runs = sorted((p for p in OUT.iterdir() if p.is_dir()), 44 key=lambda p: p.stat().st_mtime) 45 roots = runs[-1:] 46 else: 47 for pattern in patterns: 48 roots += sorted(OUT.glob(pattern)) 49 out = [] 50 for root in roots: 51 for path in sorted(root.iterdir()): 52 if path.is_dir() and (path / "summary.json").exists(): 53 out.append(path) 54 return out 55 56 57 def main() -> int: 58 ap = argparse.ArgumentParser(description=__doc__, 59 formatter_class=argparse.RawDescriptionHelpFormatter) 60 ap.add_argument("patterns", nargs="*") 61 ap.add_argument("--notes", action="store_true") 62 ap.add_argument("--kinds", action="store_true") 63 ap.add_argument("--unreached", action="store_true") 64 ap.add_argument("--evidence", action="store_true", 65 help="print each finding's evidence, not just its claim") 66 args = ap.parse_args() 67 68 paths = episodes(args.patterns) 69 if not paths: 70 print(f"no episodes under {OUT}") 71 return 1 72 73 by_kind: dict[str, list[tuple[str, dict]]] = defaultdict(list) 74 outcomes: Counter[str] = Counter() 75 total_serious = 0 76 77 for path in paths: 78 summary = json.loads((path / "summary.json").read_text()) 79 outcome = summary.get("outcome", "?") 80 outcomes[outcome] += 1 81 findings = summary.get("findings", []) 82 serious = [f for f in findings if f["kind"] not in INFORMATIONAL] 83 total_serious += len(serious) 84 for f in findings: 85 by_kind[f["kind"]].append((summary["name"], f)) 86 if args.unreached and outcome == "unreached": 87 print(f"{summary['name']} ({summary.get('seconds', 0):.0f}s)") 88 print(f" {summary.get('error', '')}") 89 for check in summary.get("checks", [])[-3:]: 90 print(f" reached: {check}") 91 continue 92 if args.unreached or args.kinds: 93 continue 94 print(f"{outcome:<14} {summary['name']:<38} " 95 f"{summary.get('seconds', 0):>6.0f}s " 96 f"{len(summary.get('checks', []))} check(s), " 97 f"{len(serious)}/{len(findings)} serious") 98 if summary.get("error"): 99 print(f" ! {summary['error'][:200]}") 100 for f in findings: 101 mark = "*" if f["kind"] not in INFORMATIONAL else "-" 102 print(f" {mark} [{f['kind']}] {f['detail'][:220]}") 103 if args.evidence and f.get("evidence"): 104 for line in f["evidence"].splitlines()[:8]: 105 print(f" {line[:200]}") 106 if args.notes: 107 for note in summary.get("notes", []): 108 print(f" . {note[:220]}") 109 print() 110 111 if args.kinds: 112 for kind, rows in sorted(by_kind.items(), key=lambda kv: -len(kv[1])): 113 mark = "*" if kind not in INFORMATIONAL else "-" 114 print(f"{mark} {kind} ({len(rows)})") 115 seen = set() 116 for name, f in rows: 117 key = f["detail"][:120] 118 if key in seen: 119 continue 120 seen.add(key) 121 print(f" {name}: {f['detail'][:200]}") 122 print() 123 124 print(f"{len(paths)} episode(s): " 125 + ", ".join(f"{n} {k}" for k, n in outcomes.most_common()) 126 + f"; {total_serious} serious finding(s)") 127 return 0 128 129 130 if __name__ == "__main__": 131 raise SystemExit(main())