search_outcomes_config.py (9990B)
1 #!/usr/bin/env python3 2 """Search an outcome file for configurations with given settings. 3 4 Read an outcome file and report the configurations in which test_suite_config 5 runs with the required settings (compilation option enabled or disabled). 6 """ 7 8 import argparse 9 import os 10 import re 11 import subprocess 12 from typing import Dict, FrozenSet, Iterator, List, Set 13 import tempfile 14 import unittest 15 16 from mbedtls_framework import build_tree 17 18 19 def make_regexp_for_settings(settings: List[str]) -> str: 20 """Construct a regexp matching the interesting outcome lines. 21 22 Interesting outcome lines are from test_suite_config where the given 23 setting is passing. 24 25 We assume that the elements of settings don't contain regexp special 26 characters. 27 """ 28 return (r';test_suite_config[^;]*;Config: (' + 29 '|'.join(settings) + 30 r');PASS;') 31 32 def run_grep(regexp: str, outcome_file: str) -> List[str]: 33 """Run grep on the outcome file and return the matching lines.""" 34 env = os.environ.copy() 35 env['LC_ALL'] = 'C' # Speeds up some versions of GNU grep 36 try: 37 return subprocess.check_output(['grep', '-E', regexp, outcome_file], 38 encoding='ascii', 39 env=env).splitlines() 40 except subprocess.CalledProcessError as exn: 41 if exn.returncode == 1: 42 return [] # No results. We don't consider this an error. 43 raise 44 45 OUTCOME_LINE_RE = re.compile(r'[^;]*;' 46 r'([^;]*);' 47 r'test_suite_config\.(?:[^;]*);' 48 r'Config: ([^;]*);' 49 r'PASS;') 50 51 def extract_configuration_data(outcome_lines: List[str]) -> Dict[str, FrozenSet[str]]: 52 """Extract the configuration data from outcome lines. 53 54 The result maps a configuration name to the list of passing settings 55 in that configuration. 56 """ 57 config_data = {} #type: Dict[str, Set[str]] 58 for line in outcome_lines: 59 m = OUTCOME_LINE_RE.match(line) 60 # Assuming a well-formed outcome file, make_regexp_for_settings() 61 # arranges to only return lines that should match OUTCOME_LINE_RE. 62 # So this assertion can't fail unless there is an unexpected 63 # divergence between OUTCOME_LINE_RE, make_regexp_for_settings() 64 # and the format of the given outcome file 65 assert m is not None 66 config_name, setting = m.groups() 67 if config_name not in config_data: 68 config_data[config_name] = set() 69 config_data[config_name].add(setting) 70 return dict((name, frozenset(settings)) 71 for name, settings in config_data.items()) 72 73 74 def matching_configurations(config_data: Dict[str, FrozenSet[str]], 75 required: List[str]) -> Iterator[str]: 76 """Search configurations with the given passing settings. 77 78 config_data maps a configuration name to the list of passing settings 79 in that configuration. 80 81 Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or 82 PSA_xxx), optionally prefixed with "!". 83 """ 84 required_set = frozenset(required) 85 for config, observed in config_data.items(): 86 if required_set.issubset(observed): 87 yield config 88 89 def search_config_outcomes(outcome_file: str, settings: List[str]) -> List[str]: 90 """Search the given outcome file for reports of the given settings. 91 92 Each setting should be an Mbed TLS compile setting (MBEDTLS_xxx or 93 PSA_xxx), optionally prefixed with "!". 94 """ 95 # The outcome file is large enough (hundreds of MB) that parsing it 96 # in Python is slow. Use grep to speed this up considerably. 97 regexp = make_regexp_for_settings(settings) 98 outcome_lines = run_grep(regexp, outcome_file) 99 config_data = extract_configuration_data(outcome_lines) 100 return sorted(matching_configurations(config_data, settings)) 101 102 103 class TestSearch(unittest.TestCase): 104 """Tests of search functionality.""" 105 106 OUTCOME_FILE_CONTENT = """\ 107 whatever;foobar;test_suite_config.part;Config: MBEDTLS_FOO;PASS; 108 whatever;foobar;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP; 109 whatever;foobar;test_suite_config.part;Config: MBEDTLS_BAR;PASS; 110 whatever;foobar;test_suite_config.part;Config: !MBEDTLS_BAR;SKIP; 111 whatever;foobar;test_suite_config.part;Config: MBEDTLS_QUX;SKIP; 112 whatever;foobar;test_suite_config.part;Config: !MBEDTLS_QUX;PASS; 113 whatever;fooqux;test_suite_config.part;Config: MBEDTLS_FOO;PASS; 114 whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_FOO;SKIP; 115 whatever;fooqux;test_suite_config.part;Config: MBEDTLS_BAR;SKIP; 116 whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_BAR;PASS; 117 whatever;fooqux;test_suite_config.part;Config: MBEDTLS_QUX;PASS; 118 whatever;fooqux;test_suite_config.part;Config: !MBEDTLS_QUX;SKIP; 119 whatever;fooqux;test_suite_something.else;Config: MBEDTLS_BAR;PASS; 120 whatever;boring;test_suite_config.part;Config: BORING;PASS; 121 whatever;parasite;not_test_suite_config.not;Config: MBEDTLS_FOO;PASS; 122 whatever;parasite;test_suite_config.but;Config: MBEDTLS_QUX with bells on;PASS; 123 whatever;parasite;test_suite_config.but;Not Config: MBEDTLS_QUX;PASS; 124 """ 125 126 def search(self, settings: List[str], expected: List[str]) -> None: 127 """Test the search functionality. 128 129 * settings: settings to search. 130 * expected: expected search results. 131 """ 132 with tempfile.NamedTemporaryFile() as tmp: 133 tmp.write(self.OUTCOME_FILE_CONTENT.encode()) 134 tmp.flush() 135 actual = search_config_outcomes(tmp.name, settings) 136 self.assertEqual(actual, expected) 137 138 def test_foo(self) -> None: 139 self.search(['MBEDTLS_FOO'], ['foobar', 'fooqux']) 140 141 def test_bar(self) -> None: 142 self.search(['MBEDTLS_BAR'], ['foobar']) 143 144 def test_foo_bar(self) -> None: 145 self.search(['MBEDTLS_FOO', 'MBEDTLS_BAR'], ['foobar']) 146 147 def test_foo_notbar(self) -> None: 148 self.search(['MBEDTLS_FOO', '!MBEDTLS_BAR'], ['fooqux']) 149 150 151 class TestOutcome(unittest.TestCase): 152 """Tests of outcome file format expectations. 153 154 This class builds and runs the config tests in the current configuration. 155 The configuration must have at least one feature enabled and at least 156 one feature disabled in each category: MBEDTLS_xxx and PSA_WANT_xxx. 157 It needs a C compiler. 158 """ 159 160 outcome_content = '' # Let mypy know this field can be used in test case methods 161 162 @classmethod 163 def setUpClass(cls) -> None: 164 """Generate, build and run the config tests.""" 165 root_dir = build_tree.guess_project_root() 166 tests_dir = os.path.join(root_dir, 'tests') 167 suites = ['test_suite_config.mbedtls_boolean', 168 'test_suite_config.psa_boolean'] 169 _output = subprocess.check_output(['make'] + suites, 170 cwd=tests_dir, 171 stderr=subprocess.STDOUT) 172 with tempfile.NamedTemporaryFile(dir=tests_dir) as outcome_file: 173 env = os.environ.copy() 174 env['MBEDTLS_TEST_PLATFORM'] = 'some_platform' 175 env['MBEDTLS_TEST_CONFIGURATION'] = 'some_configuration' 176 env['MBEDTLS_TEST_OUTCOME_FILE'] = outcome_file.name 177 for suite in suites: 178 _output = subprocess.check_output([os.path.join(os.path.curdir, suite)], 179 cwd=tests_dir, 180 env=env, 181 stderr=subprocess.STDOUT) 182 cls.outcome_content = outcome_file.read().decode('ascii') 183 184 def test_outcome_format(self) -> None: 185 """Check that there are outcome lines matching the expected general format.""" 186 def regex(prefix: str, result: str) -> str: 187 return (r'(?:\A|\n)some_platform;some_configuration;' 188 r'test_suite_config\.\w+;Config: {}_\w+;{};' 189 .format(prefix, result)) 190 self.assertRegex(self.outcome_content, regex('MBEDTLS', 'PASS')) 191 self.assertRegex(self.outcome_content, regex('MBEDTLS', 'SKIP')) 192 self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'PASS')) 193 self.assertRegex(self.outcome_content, regex('!MBEDTLS', 'SKIP')) 194 self.assertRegex(self.outcome_content, regex('PSA_WANT', 'PASS')) 195 self.assertRegex(self.outcome_content, regex('PSA_WANT', 'SKIP')) 196 self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'PASS')) 197 self.assertRegex(self.outcome_content, regex('!PSA_WANT', 'SKIP')) 198 199 def test_outcome_lines(self) -> None: 200 """Look for some sample outcome lines.""" 201 def regex(setting: str) -> str: 202 return (r'(?:\A|\n)some_platform;some_configuration;' 203 r'test_suite_config\.\w+;Config: {};(PASS|SKIP);' 204 .format(setting)) 205 self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_C')) 206 self.assertRegex(self.outcome_content, regex('MBEDTLS_AES_ROM_TABLES')) 207 self.assertRegex(self.outcome_content, regex('MBEDTLS_SSL_CLI_C')) 208 self.assertRegex(self.outcome_content, regex('MBEDTLS_X509_CRT_PARSE_C')) 209 self.assertRegex(self.outcome_content, regex('PSA_WANT_ALG_HMAC')) 210 self.assertRegex(self.outcome_content, regex('PSA_WANT_KEY_TYPE_AES')) 211 212 def main() -> None: 213 parser = argparse.ArgumentParser(description=__doc__) 214 parser.add_argument('--outcome-file', '-f', metavar='FILE', 215 default='outcomes.csv', 216 help='Outcome file to read (default: outcomes.csv)') 217 parser.add_argument('settings', metavar='SETTING', nargs='+', 218 help='Required setting (e.g. "MBEDTLS_RSA_C" or "!PSA_WANT_ALG_SHA256")') 219 options = parser.parse_args() 220 found = search_config_outcomes(options.outcome_file, options.settings) 221 for name in found: 222 print(name) 223 224 if __name__ == '__main__': 225 main()