campaign.py (8810B)
1 # This file is part of GNU Taler 2 # (C) 2026 Taler Systems S.A. 3 # 4 # GNU Taler is free software; you can redistribute it and/or modify it under the 5 # terms of the GNU General Public License as published by the Free Software 6 # Foundation; either version 3, or (at your option) any later version. 7 # 8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY 9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 10 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License along with 13 # GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 14 """Run the suite, or an exploration, across several emulators at once. 15 16 Sharding is by tag, not round-robin, and that is deliberate. The `demo` 17 scenarios all talk to the same shared `sandbox` instance, and two of them 18 running at once would see each other's orders in the history list -- so they 19 stay on one worker, and the `mock` scenarios, which each bring their own 20 backend, go on the others. 21 22 python3 campaign.py suite --workers 2 23 python3 campaign.py suite --workers 4 --tag mock 24 python3 campaign.py explore --workers 2 --steps 200 25 """ 26 27 from __future__ import annotations 28 29 import argparse 30 import json 31 import os 32 import subprocess 33 import sys 34 import time 35 from pathlib import Path 36 37 HERE = Path(__file__).resolve().parent 38 SIM_ROOT = Path(os.environ.get("POS_SIM_ROOT", "/tmp/taler-pos-sim")) 39 40 41 def scenarios_matching(tags: list[str], skip: list[str]) -> list[str]: 42 """The scenario names a tag selection covers, from the registry itself. 43 44 Imported rather than parsed out of `suite.py --list`: the listing is for 45 people, and a selection built by matching its text picked up scenarios 46 whose *description* happened to contain the tag. 47 """ 48 import suite 49 out = [] 50 for name, (_, scenario_tags, _) in suite.SCENARIOS.items(): 51 if tags and not set(tags) & set(scenario_tags): 52 continue 53 if set(skip) & set(scenario_tags): 54 continue 55 out.append(name) 56 return out 57 58 59 def shard_suite(workers: int, tags: list[str], skip: list[str]) -> list[list[str]]: 60 """The arguments for each worker's suite process. 61 62 Sharding is by tag rather than round-robin because the `demo` scenarios 63 all talk to the same shared `sandbox` instance: two of them at once would 64 see each other's orders in the history list, and one would delete an order 65 the other was waiting on. So they stay on one worker, and the `mock` 66 scenarios -- which each bring a backend of their own on a port of their 67 own -- are split across the rest. 68 """ 69 if tags: 70 # An explicit selection means the caller has said these may run beside 71 # each other, so they are split evenly. 72 chosen = scenarios_matching(tags, skip) 73 shards: list[list[str]] = [[] for _ in range(workers)] 74 for i, name in enumerate(chosen): 75 shards[i % workers].append(name) 76 return [s for s in shards if s] 77 78 plans: list[list[str]] = [["--all", "--skip-tag", "mock"]] 79 for t in skip: 80 plans[0] += ["--skip-tag", t] 81 mock = scenarios_matching(["mock"], skip) 82 # The scenarios tagged "form" are about the login form itself, and the 83 # form is unusable on a phone (findings/login-form-collapses-under-the- 84 # keyboard.md). Worker 0 is the tablet, so they go there with the demo 85 # set rather than onto a worker that can only report that they could not 86 # be reached. 87 needs_form = set(scenarios_matching(["form"], skip)) 88 if needs_form: 89 mock = [n for n in mock if n not in needs_form] 90 # Scenarios whose point is the *phone* half of a layout. Worker 0 is a 91 # tablet by convention, so running them there passes trivially and proves 92 # nothing: `login_form_fits_a_phone` is green on a tablet because the form 93 # fits. POS_PHONE_WORKERS names the workers whose AVD is phone-sized. 94 phone_workers = [int(w) for w in 95 os.environ.get("POS_PHONE_WORKERS", "1,2").split(",") 96 if w.strip().isdigit()] 97 needs_phone = set(scenarios_matching(["phone"], skip)) 98 mock_workers = max(0, workers - 1) 99 if mock_workers == 0: 100 # One worker runs everything, in one process. 101 return [["--all"] + [a for t in skip for a in ("--skip-tag", t)]] 102 for i in range(mock_workers): 103 share = mock[i::mock_workers] 104 if share: 105 plans.append(share) 106 if needs_form: 107 # Appended as explicit names after worker 0's "--all --skip-tag mock", 108 # which would otherwise leave them out for being mock scenarios. 109 plans[0] = plans[0] + sorted(needs_form) 110 if needs_phone: 111 target = next((w for w in phone_workers if 0 < w < max(workers, 1)), None) 112 if target is not None and target < len(plans): 113 plans[target] = plans[target] + sorted(needs_phone) 114 # and off the tablet, where they prove nothing 115 plans[0] = [a for a in plans[0] if a not in needs_phone] 116 plans[0] = plans[0] + ["--skip-tag", "phone"] 117 return plans[:workers] 118 119 def launch(worker: int, argv: list[str], out: Path, tag: str, 120 exclusive_demo: bool = True) -> subprocess.Popen: 121 env = dict(os.environ) 122 env["POS_WORKER"] = str(worker) 123 env["POS_MOCK_PORT"] = str(19443 + worker) 124 # `suite.py` said "the campaign sets it" and the campaign did not, so 125 # every run asserted an exclusivity nobody had established. It is set 126 # here now, and it is only true for the one worker that owns the demo 127 # plan: the others bring a mock backend of their own, and anything run by 128 # hand beside the campaign is not accounted for either way. 129 env["POS_EXCLUSIVE_DEMO"] = "1" if exclusive_demo else "0" 130 log = SIM_ROOT / "run" / f"{tag}-w{worker}.log" 131 log.parent.mkdir(parents=True, exist_ok=True) 132 handle = open(log, "w") 133 # -u matters: without it a worker's progress sits in a pipe buffer and the 134 # log stays empty for the whole run, which looks exactly like a hang. 135 proc = subprocess.Popen( 136 [sys.executable, "-u", *argv, "--out", str(out / f"w{worker}")], 137 cwd=str(HERE), env=env, stdout=handle, stderr=subprocess.STDOUT) 138 print(f"worker {worker}: {' '.join(argv[1:])} -> {log}") 139 return proc 140 141 142 def main(argv: list[str]) -> int: 143 ap = argparse.ArgumentParser(description=__doc__, 144 formatter_class=argparse.RawDescriptionHelpFormatter) 145 ap.add_argument("what", choices=["suite", "explore"]) 146 ap.add_argument("--workers", type=int, default=2) 147 ap.add_argument("--tag", action="append", default=[]) 148 ap.add_argument("--skip-tag", action="append", default=[]) 149 ap.add_argument("--steps", type=int, default=150) 150 ap.add_argument("--seed", type=int, default=None) 151 ap.add_argument("--label", default=None) 152 args = ap.parse_args(argv) 153 154 stamp = time.strftime("%Y%m%d-%H%M%S") 155 label = args.label or f"{args.what}-{stamp}" 156 out = SIM_ROOT / "out" / label 157 out.mkdir(parents=True, exist_ok=True) 158 159 procs = [] 160 if args.what == "suite": 161 shards = shard_suite(args.workers, args.tag, args.skip_tag) 162 # Worker 0 is the only one handed the demo set by `shard_suite`; an 163 # explicit --tag selection is split evenly and nobody owns it. 164 owns_demo = 0 if not args.tag else None 165 for worker, plan in enumerate(shards): 166 procs.append(launch(worker, ["suite.py", *plan], out, label, 167 exclusive_demo=(worker == owns_demo))) 168 time.sleep(2) # stagger, so two workers do not mint a token at once 169 else: 170 for worker in range(args.workers): 171 seed = (args.seed or 0) + worker if args.seed is not None else None 172 plan = ["explore.py", "--steps", str(args.steps)] 173 if seed is not None: 174 plan += ["--seed", str(seed)] 175 procs.append(launch(worker, plan, out, label)) 176 177 codes = [p.wait() for p in procs] 178 print(f"\n{len(procs)} worker(s) finished with {codes}") 179 180 merged = [] 181 for path in sorted(out.rglob("suite.json")): 182 merged += json.loads(path.read_text()) 183 if merged: 184 (out / "campaign.json").write_text(json.dumps(merged, indent=2)) 185 serious = sum(s.get("serious", 0) for s in merged) 186 unreached = [s["name"] for s in merged if s.get("outcome") == "unreached"] 187 print(f"{len(merged)} scenario(s), {serious} serious finding(s)") 188 if unreached: 189 print(f"unreached: {', '.join(unreached)}") 190 print(f"artifacts: {out}") 191 return 0 if all(c == 0 for c in codes) else 1 192 193 194 if __name__ == "__main__": 195 sys.exit(main(sys.argv[1:]))