collect_test_cases.py (6330B)
1 """Discover all the test cases (unit tests and SSL tests).""" 2 3 # Copyright The Mbed TLS Contributors 4 # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 5 6 import glob 7 import os 8 import re 9 import subprocess 10 import sys 11 12 from . import build_tree 13 14 15 class ScriptOutputError(ValueError): 16 """A kind of ValueError that indicates we found 17 the script doesn't list test cases in an expected 18 pattern. 19 """ 20 21 @property 22 def script_name(self): 23 return super().args[0] 24 25 @property 26 def idx(self): 27 return super().args[1] 28 29 @property 30 def line(self): 31 return super().args[2] 32 33 class Results: 34 """Store file and line information about errors or warnings in test suites.""" 35 36 def __init__(self, options): 37 self.errors = 0 38 self.warnings = 0 39 self.ignore_warnings = options.quiet 40 41 def error(self, file_name, line_number, fmt, *args): 42 sys.stderr.write(('{}:{}:ERROR:' + fmt + '\n'). 43 format(file_name, line_number, *args)) 44 self.errors += 1 45 46 def warning(self, file_name, line_number, fmt, *args): 47 if not self.ignore_warnings: 48 sys.stderr.write(('{}:{}:Warning:' + fmt + '\n') 49 .format(file_name, line_number, *args)) 50 self.warnings += 1 51 52 class TestDescriptionExplorer: 53 """An iterator over test cases with descriptions. 54 55 The test cases that have descriptions are: 56 * Individual unit tests (entries in a .data file) in test suites. 57 * Individual test cases in ssl-opt.sh. 58 59 This is an abstract class. To use it, derive a class that implements 60 the process_test_case method, and call walk_all(). 61 """ 62 63 def process_test_case(self, per_file_state, 64 file_name, line_number, description): 65 """Process a test case. 66 67 per_file_state: an object created by new_per_file_state() at the beginning 68 of each file. 69 file_name: a relative path to the file containing the test case. 70 line_number: the line number in the given file. 71 description: the test case description as a byte string. 72 """ 73 raise NotImplementedError 74 75 def new_per_file_state(self): 76 """Return a new per-file state object. 77 78 The default per-file state object is None. Child classes that require per-file 79 state may override this method. 80 """ 81 #pylint: disable=no-self-use 82 return None 83 84 def walk_test_suite(self, data_file_name): 85 """Iterate over the test cases in the given unit test data file.""" 86 in_paragraph = False 87 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 88 with open(data_file_name, 'rb') as data_file: 89 for line_number, line in enumerate(data_file, 1): 90 line = line.rstrip(b'\r\n') 91 if not line: 92 in_paragraph = False 93 continue 94 if line.startswith(b'#'): 95 continue 96 if not in_paragraph: 97 # This is a test case description line. 98 self.process_test_case(descriptions, 99 data_file_name, line_number, line) 100 in_paragraph = True 101 102 def collect_from_script(self, script_name): 103 """Collect the test cases in a script by calling its listing test cases 104 option""" 105 descriptions = self.new_per_file_state() # pylint: disable=assignment-from-none 106 listed = subprocess.check_output(['sh', script_name, '--list-test-cases']) 107 # Assume test file is responsible for printing identical format of 108 # test case description between --list-test-cases and its OUTCOME.CSV 109 # 110 # idx indicates the number of test case since there is no line number 111 # in the script for each test case. 112 for idx, line in enumerate(listed.splitlines()): 113 # We are expecting the script to list the test cases in 114 # `<suite_name>;<description>` pattern. 115 script_outputs = line.split(b';', 1) 116 if len(script_outputs) == 2: 117 suite_name, description = script_outputs 118 else: 119 raise ScriptOutputError(script_name, idx, line.decode("utf-8")) 120 121 self.process_test_case(descriptions, 122 suite_name.decode('utf-8'), 123 idx, 124 description.rstrip()) 125 126 @staticmethod 127 def collect_test_directories(): 128 """Get the relative path for the TLS and Crypto test directories.""" 129 project_root = build_tree.guess_project_root() 130 if build_tree.looks_like_mbedtls_root(project_root) and not build_tree.is_mbedtls_3_6(): 131 directories = [os.path.join(project_root, 'tests'), 132 os.path.join(project_root, 'tf-psa-crypto', 'tests')] 133 else: 134 directories = [os.path.join(project_root, 'tests')] 135 136 directories = [os.path.relpath(p) for p in directories] 137 return directories 138 139 def walk_all(self): 140 """Iterate over all named test cases.""" 141 test_directories = self.collect_test_directories() 142 for directory in test_directories: 143 for data_file_name in glob.glob(os.path.join(directory, 'suites', 144 '*.data')): 145 self.walk_test_suite(data_file_name) 146 147 for sh_file in ['ssl-opt.sh', 'compat.sh']: 148 sh_file = os.path.join(directory, sh_file) 149 if os.path.isfile(sh_file): 150 self.collect_from_script(sh_file) 151 152 class TestDescriptions(TestDescriptionExplorer): 153 """Collect the available test cases.""" 154 155 def __init__(self): 156 super().__init__() 157 self.descriptions = set() 158 159 def process_test_case(self, _per_file_state, 160 file_name, _line_number, description): 161 """Record an available test case.""" 162 base_name = re.sub(r'\.[^.]*$', '', re.sub(r'.*/', '', file_name)) 163 key = ';'.join([base_name, description.decode('utf-8')]) 164 self.descriptions.add(key) 165 166 def collect_available_test_cases(): 167 """Collect the available test cases.""" 168 explorer = TestDescriptions() 169 explorer.walk_all() 170 return sorted(explorer.descriptions)