matrix.py (7541B)
1 # This file is part of GNU Taler 2 # (C) 2026 Taler Systems S.A. 3 # 4 # GNU Taler is free software; you can redistribute it and/or modify it under the 5 # terms of the GNU General Public License as published by the Free Software 6 # Foundation; either version 3, or (at your option) any later version. 7 # 8 # GNU Taler is distributed in the hope that it will be useful, but WITHOUT ANY 9 # WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR 10 # A PARTICULAR PURPOSE. See the GNU General Public License for more details. 11 # 12 # You should have received a copy of the GNU General Public License along with 13 # GNU Taler; see the file COPYING. If not, see <http://www.gnu.org/licenses/> 14 """Run the platform-sensitive scenarios on every Android version the app supports. 15 16 The campaign proper runs one image (the newest), which is the wrong shape for a 17 report like "scanning works on 15 but not on 14": a defect that depends on the 18 platform cannot be seen from a single platform. 19 20 `wallet/build.gradle` says `minSdkVersion 24`, so the supported range is API 21 24 up to the compile target. Testing all thirteen is mostly waste -- what 22 matters are the levels where the app's own behaviour forks, and every entry in 23 API_LEVELS below names the code that makes it fork. 24 25 One emulator at a time, because each wants 4 GB: 26 27 python3 matrix.py # every level, the default subset 28 python3 matrix.py --apis 34 36 # just these two 29 python3 matrix.py --scenarios scan_manual_entry nfc_tap_to_pay 30 python3 matrix.py --keep # leave the last emulator running 31 32 Results land in run/matrix-<stamp>.json and are printed as a table: one row per 33 scenario, one column per API level. 34 """ 35 36 from __future__ import annotations 37 38 import argparse 39 import json 40 import os 41 import subprocess 42 import sys 43 import time 44 from pathlib import Path 45 46 HERE = Path(__file__).resolve().parent 47 SIM_ROOT = Path(os.environ.get("SIM_ROOT", "/opt/taler-sim")) 48 RUN = Path(os.environ.get("SIM_RUN_DIR", SIM_ROOT / "run")) 49 APK = Path(os.environ.get("SIM_APK", SIM_ROOT / "apk" / "wallet.apk")) 50 EMU = SIM_ROOT / "emu" / "emu.sh" 51 52 # The worker index the matrix borrows. It is past the campaign's four and past 53 # the scratch worker, so a matrix run and a campaign round can coexist. 54 MATRIX_WORKER = int(os.environ.get("SIM_MATRIX_WORKER", "6")) 55 56 # Why each level is in the list. Keep the reason with the number: without it 57 # the list grows by superstition. 58 API_LEVELS: dict[int, str] = { 59 24: "minSdkVersion -- the oldest the app claims to run on", 60 26: "TalerUtils.kt:29 branches below 26", 61 28: "SettingsScreen.kt:562 branches at P", 62 29: "Utils.kt:70 and AndroidUtils.kt:109 branch at 29 (scoped storage)", 63 30: "MainActivity.kt:301 and SettingsScreen.kt:507,554 branch at R", 64 31: "Theme.kt:158 takes dynamic colour from S", 65 33: "MainActivity.kt:360 reads NDEF the TIRAMISU way; per-app language", 66 34: "Amount.kt:256 formats with the monetary grouping separator from 34", 67 36: "the compile target, and what the campaign runs", 68 } 69 70 # The scenarios whose outcome can plausibly depend on the platform: anything 71 # touching the camera, NFC, storage, locale, biometrics or the window manager, 72 # plus enough of a money path to notice a wallet that cannot work at all. 73 DEFAULT_SCENARIOS = [ 74 "balances_screens", 75 "history", 76 "scan_manual_entry", 77 "nfc_tap_to_pay", 78 "settings_walk", 79 "settings_export", 80 "settings_import", 81 "config_changes", 82 "wallet_lock_without_credential", 83 "a11y_audit", 84 "lifecycle_kill", 85 "db_corruption", 86 ] 87 88 89 def run(cmd: list[str], **kw) -> subprocess.CompletedProcess: 90 return subprocess.run(cmd, text=True, capture_output=True, **kw) 91 92 93 def emu(action: str, *args: str, api: int | None = None, timeout: int = 900): 94 env = dict(os.environ) 95 if api is not None: 96 env["SIM_SYSTEM_IMAGE"] = f"system-images;android-{api};google_apis;x86_64" 97 return subprocess.run( 98 [str(EMU), action, str(MATRIX_WORKER), *args], 99 text=True, capture_output=True, env=env, timeout=timeout, 100 ) 101 102 103 def boot(api: int) -> str: 104 """Create, boot and prepare one emulator on this API level.""" 105 serial = f"emulator-{5554 + 2 * MATRIX_WORKER}" 106 print(f" creating an API {api} emulator ...", flush=True) 107 made = emu("create", api=api) 108 if made.returncode != 0: 109 raise RuntimeError(f"could not create the AVD: {made.stderr.strip()[:300]}") 110 print(f" booting {serial} ...", flush=True) 111 booted = emu("boot", api=api, timeout=1200) 112 if booted.returncode != 0: 113 raise RuntimeError(f"could not boot: {booted.stderr.strip()[:300]}") 114 prepared = emu("prepare", str(APK), api=api, timeout=900) 115 if prepared.returncode != 0: 116 raise RuntimeError(f"could not prepare: {prepared.stderr.strip()[:300]}") 117 return serial 118 119 120 def scenarios_on(api: int, scenarios: list[str], timeout: int) -> dict: 121 """Run the subset and return each scenario's status.""" 122 env = dict(os.environ, SIM_WORKER=str(MATRIX_WORKER)) 123 log = RUN / f"matrix-api{api}.log" 124 with open(log, "w") as out: 125 proc = subprocess.run( 126 [sys.executable, "-u", str(HERE / "suite.py"), *scenarios], 127 text=True, stdout=out, stderr=subprocess.STDOUT, 128 env=env, cwd=HERE, timeout=timeout, 129 ) 130 results: dict[str, str] = {} 131 for line in log.read_text(errors="replace").splitlines(): 132 parts = line.split() 133 if len(parts) >= 2 and parts[0] in scenarios: 134 results[parts[0]] = parts[1] 135 return {"exit": proc.returncode, "log": str(log), "scenarios": results} 136 137 138 def main() -> int: 139 ap = argparse.ArgumentParser(description=__doc__) 140 ap.add_argument("--apis", nargs="*", type=int, default=sorted(API_LEVELS), 141 help="API levels to run (default: all of them)") 142 ap.add_argument("--scenarios", nargs="*", default=DEFAULT_SCENARIOS) 143 ap.add_argument("--timeout", type=int, default=5400, 144 help="seconds for one level's scenarios") 145 ap.add_argument("--keep", action="store_true", 146 help="leave the last emulator running") 147 args = ap.parse_args() 148 149 stamp = int(time.time()) 150 out_path = RUN / f"matrix-{stamp}.json" 151 report: dict = {"stamp": stamp, "scenarios": args.scenarios, "levels": {}} 152 153 for api in args.apis: 154 print(f"=== API {api}: {API_LEVELS.get(api, 'no reason recorded')}", flush=True) 155 try: 156 boot(api) 157 report["levels"][api] = scenarios_on(api, args.scenarios, args.timeout) 158 except Exception as e: # noqa: BLE001 159 print(f" !! {e}", flush=True) 160 report["levels"][api] = {"error": str(e)[:400], "scenarios": {}} 161 finally: 162 if not (args.keep and api == args.apis[-1]): 163 emu("kill", api=api, timeout=120) 164 out_path.write_text(json.dumps(report, indent=2)) 165 166 # One row per scenario, one column per level: a difference between columns 167 # is the thing this exists to find. 168 levels = [a for a in args.apis if a in report["levels"]] 169 width = max((len(s) for s in args.scenarios), default=10) + 2 170 print("\n" + "scenario".ljust(width) + "".join(f"{a:>7}" for a in levels)) 171 for name in args.scenarios: 172 row = "".join( 173 f"{report['levels'][a]['scenarios'].get(name, '-')[:6]:>7}" 174 for a in levels 175 ) 176 print(name.ljust(width) + row) 177 print(f"\n{out_path}") 178 return 0 179 180 181 if __name__ == "__main__": 182 sys.exit(main())