test_data_generation.py (8827B)
1 """Common code for test data generation. 2 3 This module defines classes that are of general use to automatically 4 generate .data files for unit tests, as well as a main function. 5 6 These are used both by generate_psa_tests.py and generate_bignum_tests.py. 7 """ 8 9 # Copyright The Mbed TLS Contributors 10 # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 11 # 12 13 import argparse 14 import os 15 import posixpath 16 import re 17 import inspect 18 19 from abc import ABCMeta, abstractmethod 20 from typing import Callable, Dict, Iterable, Iterator, List, Type, TypeVar 21 22 from . import build_tree 23 from . import test_case 24 25 T = TypeVar('T') #pylint: disable=invalid-name 26 27 28 class BaseTest(metaclass=ABCMeta): 29 """Base class for test case generation. 30 31 Attributes: 32 count: Counter for test cases from this class. 33 case_description: Short description of the test case. This may be 34 automatically generated using the class, or manually set. 35 dependencies: A list of dependencies required for the test case. 36 show_test_count: Toggle for inclusion of `count` in the test description. 37 test_function: Test function which the class generates cases for. 38 test_name: A common name or description of the test function. This can 39 be `test_function`, a clearer equivalent, or a short summary of the 40 test function's purpose. 41 """ 42 count = 0 43 case_description = "" 44 dependencies = [] # type: List[str] 45 show_test_count = True 46 test_function = "" 47 test_name = "" 48 49 def __new__(cls, *args, **kwargs): 50 # pylint: disable=unused-argument 51 cls.count += 1 52 return super().__new__(cls) 53 54 @abstractmethod 55 def arguments(self) -> List[str]: 56 """Get the list of arguments for the test case. 57 58 Override this method to provide the list of arguments required for 59 the `test_function`. 60 61 Returns: 62 List of arguments required for the test function. 63 """ 64 raise NotImplementedError 65 66 def description(self) -> str: 67 """Create a test case description. 68 69 Creates a description of the test case, including a name for the test 70 function, an optional case count, and a description of the specific 71 test case. This should inform a reader what is being tested, and 72 provide context for the test case. 73 74 Returns: 75 Description for the test case. 76 """ 77 if self.show_test_count: 78 return "{} #{} {}".format( 79 self.test_name, self.count, self.case_description 80 ).strip() 81 else: 82 return "{} {}".format(self.test_name, self.case_description).strip() 83 84 85 def create_test_case(self) -> test_case.TestCase: 86 """Generate TestCase from the instance.""" 87 tc = test_case.TestCase() 88 tc.set_description(self.description()) 89 tc.set_function(self.test_function) 90 tc.set_arguments(self.arguments()) 91 tc.set_dependencies(self.dependencies) 92 93 return tc 94 95 @classmethod 96 @abstractmethod 97 def generate_function_tests(cls) -> Iterator[test_case.TestCase]: 98 """Generate test cases for the class test function. 99 100 This will be called in classes where `test_function` is set. 101 Implementations should yield TestCase objects, by creating instances 102 of the class with appropriate input data, and then calling 103 `create_test_case()` on each. 104 """ 105 raise NotImplementedError 106 107 108 class BaseTarget: 109 #pylint: disable=too-few-public-methods 110 """Base target for test case generation. 111 112 Child classes of this class represent an output file, and can be referred 113 to as file targets. These indicate where test cases will be written to for 114 all subclasses of the file target, which is set by `target_basename`. 115 116 Attributes: 117 target_basename: Basename of file to write generated tests to. This 118 should be specified in a child class of BaseTarget. 119 """ 120 target_basename = "" 121 122 @classmethod 123 def generate_tests(cls) -> Iterator[test_case.TestCase]: 124 """Generate test cases for the class and its subclasses. 125 126 In classes with `test_function` set, `generate_function_tests()` is 127 called to generate test cases first. 128 129 In all classes, this method will iterate over its subclasses, and 130 yield from `generate_tests()` in each. Calling this method on a class X 131 will yield test cases from all classes derived from X. 132 """ 133 if issubclass(cls, BaseTest) and not inspect.isabstract(cls): 134 #pylint: disable=no-member 135 yield from cls.generate_function_tests() 136 for subclass in sorted(cls.__subclasses__(), key=lambda c: c.__name__): 137 yield from subclass.generate_tests() 138 139 140 class TestGenerator: 141 """Generate test cases and write to data files.""" 142 def __init__(self, options) -> None: 143 self.test_suite_directory = options.directory 144 # Update `targets` with an entry for each child class of BaseTarget. 145 # Each entry represents a file generated by the BaseTarget framework, 146 # and enables generating the .data files using the CLI. 147 self.targets.update({ 148 subclass.target_basename: subclass.generate_tests 149 for subclass in BaseTarget.__subclasses__() 150 if subclass.target_basename 151 }) 152 153 def filename_for(self, basename: str) -> str: 154 """The location of the data file with the specified base name.""" 155 return posixpath.join(self.test_suite_directory, basename + '.data') 156 157 def write_test_data_file(self, basename: str, 158 test_cases: Iterable[test_case.TestCase]) -> None: 159 """Write the test cases to a .data file. 160 161 The output file is ``basename + '.data'`` in the test suite directory. 162 """ 163 filename = self.filename_for(basename) 164 test_case.write_data_file(filename, test_cases) 165 166 # Note that targets whose names contain 'test_format' have their content 167 # validated by `abi_check.py`. 168 targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]] 169 170 def generate_target(self, name: str, *target_args) -> None: 171 """Generate cases and write to data file for a target. 172 173 For target callables which require arguments, override this function 174 and pass these arguments using super() (see PSATestGenerator). 175 """ 176 test_cases = self.targets[name](*target_args) 177 self.write_test_data_file(name, test_cases) 178 179 def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator): 180 """Command line entry point.""" 181 parser = argparse.ArgumentParser(description=description) 182 parser.add_argument('--list', action='store_true', 183 help='List available targets and exit') 184 parser.add_argument('--list-for-cmake', action='store_true', 185 help='Print \';\'-separated list of available targets and exit') 186 # If specified explicitly, this option may be a path relative to the 187 # current directory when the script is invoked. The default value 188 # is relative to the mbedtls root, which we don't know yet. So we 189 # can't set a string as the default value here. 190 parser.add_argument('--directory', metavar='DIR', 191 help='Output directory (default: tests/suites)') 192 parser.add_argument('targets', nargs='*', metavar='TARGET', 193 help='Target file to generate (default: all; "-": none)') 194 options = parser.parse_args(args) 195 196 # Change to the mbedtls root, to keep things simple. But first, adjust 197 # command line options that might be relative paths. 198 if options.directory is None: 199 options.directory = 'tests/suites' 200 else: 201 options.directory = os.path.abspath(options.directory) 202 build_tree.chdir_to_root() 203 204 generator = generator_class(options) 205 if options.list: 206 for name in sorted(generator.targets): 207 print(generator.filename_for(name)) 208 return 209 # List in a cmake list format (i.e. ';'-separated) 210 if options.list_for_cmake: 211 print(';'.join(generator.filename_for(name) 212 for name in sorted(generator.targets)), end='') 213 return 214 if options.targets: 215 # Allow "-" as a special case so you can run 216 # ``generate_xxx_tests.py - $targets`` and it works uniformly whether 217 # ``$targets`` is empty or not. 218 options.targets = [os.path.basename(re.sub(r'\.data\Z', r'', target)) 219 for target in options.targets 220 if target != '-'] 221 else: 222 options.targets = sorted(generator.targets) 223 for target in options.targets: 224 generator.generate_target(target)