taler-android

Android apps for GNU Taler (wallet, PoS, cashier)
Log | Files | Refs | README | LICENSE

campaign.py (7735B)


      1 # This file is part of GNU Taler
      2 # (C) 2026 Taler Systems S.A.
      3 #
      4 # GNU Taler is free software; you can redistribute it and/or modify it under the
      5 # terms of the GNU General Public License as published by the Free Software
      6 # Foundation; either version 3, or (at your option) any later version.
      7 #
      8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY
      9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
     10 # A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
     11 #
     12 # You should have received a copy of the GNU General Public License along with
     13 # GNU Taler; see the file COPYING.  If not, see <http://www.gnu.org/licenses/>
     14 """Run scenarios and explorations across several emulators at once.
     15 
     16 Each worker has its own emulator, its own bank customer and its own CLI peer
     17 wallet, so they only share the Taler services.  Work is split round-robin and
     18 the per-worker summaries are merged into one report.
     19 
     20     python3 campaign.py suite                 # every scenario, 4 ways
     21     python3 campaign.py suite --workers 2
     22     python3 campaign.py explore --steps 150   # one exploration per worker
     23     python3 campaign.py both
     24 """
     25 
     26 from __future__ import annotations
     27 
     28 import argparse
     29 import json
     30 import os
     31 import subprocess
     32 import time
     33 from collections.abc import Iterable
     34 from pathlib import Path
     35 
     36 import oracles
     37 import suite
     38 from device import Device
     39 
     40 OUT = Path("/opt/taler-sim/run/out")
     41 COVERAGE = Path("/opt/taler-sim/run/coverage")
     42 HERE = Path(__file__).resolve().parent
     43 
     44 
     45 def online_workers(limit: int) -> list[int]:
     46     """Workers whose emulator is actually up."""
     47     listed = subprocess.run(
     48         ["/opt/android-sdk/platform-tools/adb", "devices"],
     49         capture_output=True, text=True, timeout=60,
     50     ).stdout
     51     found = []
     52     for worker in range(limit):
     53         if f"emulator-{5554 + 2 * worker}\tdevice" in listed:
     54             found.append(worker)
     55     return found
     56 
     57 
     58 def split(names: list[str], workers: list[int]) -> dict[int, list[str]]:
     59     buckets: dict[int, list[str]] = {w: [] for w in workers}
     60     for i, name in enumerate(names):
     61         buckets[workers[i % len(workers)]].append(name)
     62     return buckets
     63 
     64 
     65 def launch(worker: int, args: list[str], tag: str) -> tuple[int, subprocess.Popen, Path]:
     66     env = dict(os.environ, SIM_WORKER=str(worker))
     67     log = OUT / f"{tag}-w{worker}.log"
     68     log.parent.mkdir(parents=True, exist_ok=True)
     69     handle = log.open("w")
     70     proc = subprocess.Popen(args, cwd=HERE, env=env, stdout=handle,
     71                             stderr=subprocess.STDOUT, text=True)
     72     return worker, proc, log
     73 
     74 
     75 def wait_all(jobs: list[tuple[int, subprocess.Popen, Path]], poll: int = 20) -> None:
     76     pending = {w: (p, log) for w, p, log in jobs}
     77     while pending:
     78         for worker in list(pending):
     79             proc, log = pending[worker]
     80             if proc.poll() is not None:
     81                 print(f"worker {worker} finished with code {proc.returncode} ({log})")
     82                 del pending[worker]
     83         if pending:
     84             time.sleep(poll)
     85 
     86 
     87 def collect(stamp_after: float, workers: Iterable[int] | None = None) -> dict:
     88     """Merge the per-worker suite summaries this run produced.
     89 
     90     Filtered by worker as well as by time: a scenario run by hand on a spare
     91     emulator writes a summary into the same directory, and round 20's tally
     92     swallowed three such runs -- 147 "scenarios", and findings from a patched
     93     APK reported against an unpatched round.
     94     """
     95     wanted = None if workers is None else {f"suite-w{w}-" for w in workers}
     96     merged: dict = {"scenarios": [], "findings": {}}
     97     for path in sorted(OUT.glob("suite-w*-summary.json")):
     98         if path.stat().st_mtime < stamp_after:
     99             continue
    100         if wanted is not None and not any(path.name.startswith(p) for p in wanted):
    101             continue
    102         data = json.loads(path.read_text())
    103         merged["scenarios"] += data.get("scenarios", [])
    104         for f in data.get("findings", []):
    105             merged["findings"][f["fingerprint"]] = f
    106     return merged
    107 
    108 
    109 def report(merged: dict) -> None:
    110     rows = merged["scenarios"]
    111     ok = [r for r in rows if r.get("status") == "ok"]
    112     broken = [r for r in rows if r.get("status") != "ok"]
    113     print(f"\n{len(rows)} scenarios: {len(ok)} ok, {len(broken)} driver errors")
    114     for r in sorted(rows, key=lambda r: r.get("scenario") or ""):
    115         flag = " " if r.get("status") == "ok" else "!"
    116         print(f" {flag} {r.get('scenario'):26} {r.get('serious', 0)} serious, "
    117               f"{len(r.get('findings', []))} findings, {r.get('seconds')}s")
    118         if r.get("error"):
    119             print(f"     error: {r['error'][:200]}")
    120     # oracles.INFORMATIONAL, the same rule the session and the suite use: a
    121     # withdrawal the suite leaves pending on purpose is not a defect.
    122     serious = [f for f in merged["findings"].values()
    123                if f["kind"] not in oracles.INFORMATIONAL]
    124     print(f"\nfindings: {len(merged['findings'])} distinct, {len(serious)} serious")
    125     for f in merged["findings"].values():
    126         mark = "***" if f["kind"] not in oracles.INFORMATIONAL else "   "
    127         print(f" {mark} [{f['kind']}] {f['fingerprint']} {f['detail'][:170]}")
    128 
    129 
    130 def dump_coverage(workers: list[int], tag: str) -> None:
    131     import coverage as coverage_mod
    132     for worker in workers:
    133         dev = Device(f"emulator-{5554 + 2 * worker}")
    134         try:
    135             path = coverage_mod.dump(dev, COVERAGE / f"{tag}-w{worker}.ec", name=f"{tag}.ec")
    136             print(f"coverage worker {worker}: {path} ({path.stat().st_size} bytes)")
    137         except Exception as e:
    138             print(f"coverage worker {worker} failed: {e}")
    139 
    140 
    141 def main() -> int:
    142     ap = argparse.ArgumentParser(description=__doc__)
    143     ap.add_argument("mode", choices=["suite", "explore", "both"])
    144     ap.add_argument("--workers", type=int, default=4)
    145     ap.add_argument("--steps", type=int, default=120, help="explore steps per worker")
    146     ap.add_argument("--seed", type=int, default=100)
    147     ap.add_argument("--only", nargs="*", help="scenario names (suite mode)")
    148     ap.add_argument("--tag", default=None)
    149     args = ap.parse_args()
    150 
    151     workers = online_workers(args.workers)
    152     if not workers:
    153         print("no emulators online")
    154         return 1
    155     tag = args.tag or f"{args.mode}-{int(time.time())}"
    156     started = time.time()
    157     print(f"workers online: {workers}  tag: {tag}")
    158 
    159     if args.mode in ("suite", "both"):
    160         names = args.only or list(suite.REGISTRY)
    161         buckets = split(names, workers)
    162         jobs = []
    163         for worker, bucket in buckets.items():
    164             if not bucket:
    165                 continue
    166             print(f"worker {worker}: {len(bucket)} scenarios -> {bucket}")
    167             jobs.append(launch(worker, ["python3", "-u", "suite.py", *bucket],
    168                                f"{tag}-suite"))
    169         wait_all(jobs)
    170         dump_coverage(list(buckets), f"{tag}-suite")
    171         report(collect(started, buckets))
    172 
    173     if args.mode in ("explore", "both"):
    174         started = time.time()
    175         jobs = []
    176         for worker in workers:
    177             seed = args.seed + worker
    178             print(f"worker {worker}: explore {args.steps} steps, seed {seed}")
    179             jobs.append(launch(worker, ["python3", "-u", "explore.py",
    180                                         str(args.steps), str(seed)], f"{tag}-explore"))
    181         wait_all(jobs)
    182         dump_coverage(workers, f"{tag}-explore")
    183         for worker in workers:
    184             log = OUT / f"{tag}-explore-w{worker}.log"
    185             if log.exists():
    186                 print(f"\n--- worker {worker} ---")
    187                 print(log.read_text()[-1500:])
    188     return 0
    189 
    190 
    191 if __name__ == "__main__":
    192     raise SystemExit(main())