campaign.py (7735B)
1 # This file is part of GNU Taler 2 # (C) 2026 Taler Systems S.A. 3 # 4 # GNU Taler is free software; you can redistribute it and/or modify it under the 5 # terms of the GNU General Public License as published by the Free Software 6 # Foundation; either version 3, or (at your option) any later version. 7 # 8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY 9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 10 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License along with 13 # GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 14 """Run scenarios and explorations across several emulators at once. 15 16 Each worker has its own emulator, its own bank customer and its own CLI peer 17 wallet, so they only share the Taler services. Work is split round-robin and 18 the per-worker summaries are merged into one report. 19 20 python3 campaign.py suite # every scenario, 4 ways 21 python3 campaign.py suite --workers 2 22 python3 campaign.py explore --steps 150 # one exploration per worker 23 python3 campaign.py both 24 """ 25 26 from __future__ import annotations 27 28 import argparse 29 import json 30 import os 31 import subprocess 32 import time 33 from collections.abc import Iterable 34 from pathlib import Path 35 36 import oracles 37 import suite 38 from device import Device 39 40 OUT = Path("/opt/taler-sim/run/out") 41 COVERAGE = Path("/opt/taler-sim/run/coverage") 42 HERE = Path(__file__).resolve().parent 43 44 45 def online_workers(limit: int) -> list[int]: 46 """Workers whose emulator is actually up.""" 47 listed = subprocess.run( 48 ["/opt/android-sdk/platform-tools/adb", "devices"], 49 capture_output=True, text=True, timeout=60, 50 ).stdout 51 found = [] 52 for worker in range(limit): 53 if f"emulator-{5554 + 2 * worker}\tdevice" in listed: 54 found.append(worker) 55 return found 56 57 58 def split(names: list[str], workers: list[int]) -> dict[int, list[str]]: 59 buckets: dict[int, list[str]] = {w: [] for w in workers} 60 for i, name in enumerate(names): 61 buckets[workers[i % len(workers)]].append(name) 62 return buckets 63 64 65 def launch(worker: int, args: list[str], tag: str) -> tuple[int, subprocess.Popen, Path]: 66 env = dict(os.environ, SIM_WORKER=str(worker)) 67 log = OUT / f"{tag}-w{worker}.log" 68 log.parent.mkdir(parents=True, exist_ok=True) 69 handle = log.open("w") 70 proc = subprocess.Popen(args, cwd=HERE, env=env, stdout=handle, 71 stderr=subprocess.STDOUT, text=True) 72 return worker, proc, log 73 74 75 def wait_all(jobs: list[tuple[int, subprocess.Popen, Path]], poll: int = 20) -> None: 76 pending = {w: (p, log) for w, p, log in jobs} 77 while pending: 78 for worker in list(pending): 79 proc, log = pending[worker] 80 if proc.poll() is not None: 81 print(f"worker {worker} finished with code {proc.returncode} ({log})") 82 del pending[worker] 83 if pending: 84 time.sleep(poll) 85 86 87 def collect(stamp_after: float, workers: Iterable[int] | None = None) -> dict: 88 """Merge the per-worker suite summaries this run produced. 89 90 Filtered by worker as well as by time: a scenario run by hand on a spare 91 emulator writes a summary into the same directory, and round 20's tally 92 swallowed three such runs -- 147 "scenarios", and findings from a patched 93 APK reported against an unpatched round. 94 """ 95 wanted = None if workers is None else {f"suite-w{w}-" for w in workers} 96 merged: dict = {"scenarios": [], "findings": {}} 97 for path in sorted(OUT.glob("suite-w*-summary.json")): 98 if path.stat().st_mtime < stamp_after: 99 continue 100 if wanted is not None and not any(path.name.startswith(p) for p in wanted): 101 continue 102 data = json.loads(path.read_text()) 103 merged["scenarios"] += data.get("scenarios", []) 104 for f in data.get("findings", []): 105 merged["findings"][f["fingerprint"]] = f 106 return merged 107 108 109 def report(merged: dict) -> None: 110 rows = merged["scenarios"] 111 ok = [r for r in rows if r.get("status") == "ok"] 112 broken = [r for r in rows if r.get("status") != "ok"] 113 print(f"\n{len(rows)} scenarios: {len(ok)} ok, {len(broken)} driver errors") 114 for r in sorted(rows, key=lambda r: r.get("scenario") or ""): 115 flag = " " if r.get("status") == "ok" else "!" 116 print(f" {flag} {r.get('scenario'):26} {r.get('serious', 0)} serious, " 117 f"{len(r.get('findings', []))} findings, {r.get('seconds')}s") 118 if r.get("error"): 119 print(f" error: {r['error'][:200]}") 120 # oracles.INFORMATIONAL, the same rule the session and the suite use: a 121 # withdrawal the suite leaves pending on purpose is not a defect. 122 serious = [f for f in merged["findings"].values() 123 if f["kind"] not in oracles.INFORMATIONAL] 124 print(f"\nfindings: {len(merged['findings'])} distinct, {len(serious)} serious") 125 for f in merged["findings"].values(): 126 mark = "***" if f["kind"] not in oracles.INFORMATIONAL else " " 127 print(f" {mark} [{f['kind']}] {f['fingerprint']} {f['detail'][:170]}") 128 129 130 def dump_coverage(workers: list[int], tag: str) -> None: 131 import coverage as coverage_mod 132 for worker in workers: 133 dev = Device(f"emulator-{5554 + 2 * worker}") 134 try: 135 path = coverage_mod.dump(dev, COVERAGE / f"{tag}-w{worker}.ec", name=f"{tag}.ec") 136 print(f"coverage worker {worker}: {path} ({path.stat().st_size} bytes)") 137 except Exception as e: 138 print(f"coverage worker {worker} failed: {e}") 139 140 141 def main() -> int: 142 ap = argparse.ArgumentParser(description=__doc__) 143 ap.add_argument("mode", choices=["suite", "explore", "both"]) 144 ap.add_argument("--workers", type=int, default=4) 145 ap.add_argument("--steps", type=int, default=120, help="explore steps per worker") 146 ap.add_argument("--seed", type=int, default=100) 147 ap.add_argument("--only", nargs="*", help="scenario names (suite mode)") 148 ap.add_argument("--tag", default=None) 149 args = ap.parse_args() 150 151 workers = online_workers(args.workers) 152 if not workers: 153 print("no emulators online") 154 return 1 155 tag = args.tag or f"{args.mode}-{int(time.time())}" 156 started = time.time() 157 print(f"workers online: {workers} tag: {tag}") 158 159 if args.mode in ("suite", "both"): 160 names = args.only or list(suite.REGISTRY) 161 buckets = split(names, workers) 162 jobs = [] 163 for worker, bucket in buckets.items(): 164 if not bucket: 165 continue 166 print(f"worker {worker}: {len(bucket)} scenarios -> {bucket}") 167 jobs.append(launch(worker, ["python3", "-u", "suite.py", *bucket], 168 f"{tag}-suite")) 169 wait_all(jobs) 170 dump_coverage(list(buckets), f"{tag}-suite") 171 report(collect(started, buckets)) 172 173 if args.mode in ("explore", "both"): 174 started = time.time() 175 jobs = [] 176 for worker in workers: 177 seed = args.seed + worker 178 print(f"worker {worker}: explore {args.steps} steps, seed {seed}") 179 jobs.append(launch(worker, ["python3", "-u", "explore.py", 180 str(args.steps), str(seed)], f"{tag}-explore")) 181 wait_all(jobs) 182 dump_coverage(workers, f"{tag}-explore") 183 for worker in workers: 184 log = OUT / f"{tag}-explore-w{worker}.log" 185 if log.exists(): 186 print(f"\n--- worker {worker} ---") 187 print(log.read_text()[-1500:]) 188 return 0 189 190 191 if __name__ == "__main__": 192 raise SystemExit(main())