psa_collect_statuses.py (5214B)
1 #!/usr/bin/env python3 2 """Describe the test coverage of PSA functions in terms of return statuses. 3 4 1. Build Mbed TLS with -DRECORD_PSA_STATUS_COVERAGE_LOG 5 2. Run psa_collect_statuses.py 6 7 The output is a series of line of the form "psa_foo PSA_ERROR_XXX". Each 8 function/status combination appears only once. 9 10 This script must be run from the top of an Mbed TLS source tree. 11 The build command is "make -DRECORD_PSA_STATUS_COVERAGE_LOG", which is 12 only supported with make (as opposed to CMake or other build methods). 13 """ 14 15 # Copyright The Mbed TLS Contributors 16 # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 17 18 import argparse 19 import os 20 import subprocess 21 import sys 22 23 DEFAULT_STATUS_LOG_FILE = 'tests/statuses.log' 24 DEFAULT_PSA_CONSTANT_NAMES = 'programs/psa/psa_constant_names' 25 26 class Statuses: 27 """Information about observed return statues of API functions.""" 28 29 def __init__(self): 30 self.functions = {} 31 self.codes = set() 32 self.status_names = {} 33 34 def collect_log(self, log_file_name): 35 """Read logs from RECORD_PSA_STATUS_COVERAGE_LOG. 36 37 Read logs produced by running Mbed TLS test suites built with 38 -DRECORD_PSA_STATUS_COVERAGE_LOG. 39 """ 40 with open(log_file_name) as log: 41 for line in log: 42 value, function, tail = line.split(':', 2) 43 if function not in self.functions: 44 self.functions[function] = {} 45 fdata = self.functions[function] 46 if value not in self.functions[function]: 47 fdata[value] = [] 48 fdata[value].append(tail) 49 self.codes.add(int(value)) 50 51 def get_constant_names(self, psa_constant_names): 52 """Run psa_constant_names to obtain names for observed numerical values.""" 53 values = [str(value) for value in self.codes] 54 cmd = [psa_constant_names, 'status'] + values 55 output = subprocess.check_output(cmd).decode('ascii') 56 for value, name in zip(values, output.rstrip().split('\n')): 57 self.status_names[value] = name 58 59 def report(self): 60 """Report observed return values for each function. 61 62 The report is a series of line of the form "psa_foo PSA_ERROR_XXX". 63 """ 64 for function in sorted(self.functions.keys()): 65 fdata = self.functions[function] 66 names = [self.status_names[value] for value in fdata.keys()] 67 for name in sorted(names): 68 sys.stdout.write('{} {}\n'.format(function, name)) 69 70 def collect_status_logs(options): 71 """Build and run unit tests and report observed function return statuses. 72 73 Build Mbed TLS with -DRECORD_PSA_STATUS_COVERAGE_LOG, run the 74 test suites and display information about observed return statuses. 75 """ 76 rebuilt = False 77 if not options.use_existing_log and os.path.exists(options.log_file): 78 os.remove(options.log_file) 79 if not os.path.exists(options.log_file): 80 if options.clean_before: 81 subprocess.check_call(['make', 'clean'], 82 cwd='tests', 83 stdout=sys.stderr) 84 with open(os.devnull, 'w') as devnull: 85 make_q_ret = subprocess.call(['make', '-q', 'lib', 'tests'], 86 stdout=devnull, stderr=devnull) 87 if make_q_ret != 0: 88 subprocess.check_call(['make', 'RECORD_PSA_STATUS_COVERAGE_LOG=1'], 89 stdout=sys.stderr) 90 rebuilt = True 91 subprocess.check_call(['make', 'test'], 92 stdout=sys.stderr) 93 data = Statuses() 94 data.collect_log(options.log_file) 95 data.get_constant_names(options.psa_constant_names) 96 if rebuilt and options.clean_after: 97 subprocess.check_call(['make', 'clean'], 98 cwd='tests', 99 stdout=sys.stderr) 100 return data 101 102 def main(): 103 parser = argparse.ArgumentParser(description=globals()['__doc__']) 104 parser.add_argument('--clean-after', 105 action='store_true', 106 help='Run "make clean" after rebuilding') 107 parser.add_argument('--clean-before', 108 action='store_true', 109 help='Run "make clean" before regenerating the log file)') 110 parser.add_argument('--log-file', metavar='FILE', 111 default=DEFAULT_STATUS_LOG_FILE, 112 help='Log file location (default: {})'.format( 113 DEFAULT_STATUS_LOG_FILE 114 )) 115 parser.add_argument('--psa-constant-names', metavar='PROGRAM', 116 default=DEFAULT_PSA_CONSTANT_NAMES, 117 help='Path to psa_constant_names (default: {})'.format( 118 DEFAULT_PSA_CONSTANT_NAMES 119 )) 120 parser.add_argument('--use-existing-log', '-e', 121 action='store_true', 122 help='Don\'t regenerate the log file if it exists') 123 options = parser.parse_args() 124 data = collect_status_logs(options) 125 data.report() 126 127 if __name__ == '__main__': 128 main()