test_config_script.py (6920B)
1 #!/usr/bin/env python3 2 3 """Test helper for the Mbed TLS configuration file tool 4 5 Run config.py with various parameters and write the results to files. 6 7 This is a harness to help regression testing, not a functional tester. 8 Sample usage: 9 10 test_config_script.py -d old 11 ## Modify config.py and/or mbedtls_config.h ## 12 test_config_script.py -d new 13 diff -ru old new 14 """ 15 16 ## Copyright The Mbed TLS Contributors 17 ## SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 18 ## 19 20 import argparse 21 import glob 22 import os 23 import re 24 import shutil 25 import subprocess 26 27 OUTPUT_FILE_PREFIX = 'config-' 28 29 def output_file_name(directory, stem, extension): 30 return os.path.join(directory, 31 '{}{}.{}'.format(OUTPUT_FILE_PREFIX, 32 stem, extension)) 33 34 def cleanup_directory(directory): 35 """Remove old output files.""" 36 for extension in []: 37 pattern = output_file_name(directory, '*', extension) 38 filenames = glob.glob(pattern) 39 for filename in filenames: 40 os.remove(filename) 41 42 def prepare_directory(directory): 43 """Create the output directory if it doesn't exist yet. 44 45 If there are old output files, remove them. 46 """ 47 if os.path.exists(directory): 48 cleanup_directory(directory) 49 else: 50 os.makedirs(directory) 51 52 def guess_presets_from_help(help_text): 53 """Figure out what presets the script supports. 54 55 help_text should be the output from running the script with --help. 56 """ 57 # Try the output format from config.py 58 hits = re.findall(r'\{([-\w,]+)\}', help_text) 59 for hit in hits: 60 words = set(hit.split(',')) 61 if 'get' in words and 'set' in words and 'unset' in words: 62 words.remove('get') 63 words.remove('set') 64 words.remove('unset') 65 return words 66 # Try the output format from config.pl 67 hits = re.findall(r'\n +([-\w]+) +- ', help_text) 68 if hits: 69 return hits 70 raise Exception("Unable to figure out supported presets. Pass the '-p' option.") 71 72 def list_presets(options): 73 """Return the list of presets to test. 74 75 The list is taken from the command line if present, otherwise it is 76 extracted from running the config script with --help. 77 """ 78 if options.presets: 79 return re.split(r'[ ,]+', options.presets) 80 else: 81 help_text = subprocess.run([options.script, '--help'], 82 check=False, # config.pl --help returns 255 83 stdout=subprocess.PIPE, 84 stderr=subprocess.STDOUT).stdout 85 return guess_presets_from_help(help_text.decode('ascii')) 86 87 def run_one(options, args, stem_prefix='', input_file=None): 88 """Run the config script with the given arguments. 89 90 Take the original content from input_file if specified, defaulting 91 to options.input_file if input_file is None. 92 93 Write the following files, where xxx contains stem_prefix followed by 94 a filename-friendly encoding of args: 95 * config-xxx.h: modified file. 96 * config-xxx.out: standard output. 97 * config-xxx.err: standard output. 98 * config-xxx.status: exit code. 99 100 Return ("xxx+", "path/to/config-xxx.h") which can be used as 101 stem_prefix and input_file to call this function again with new args. 102 """ 103 if input_file is None: 104 input_file = options.input_file 105 stem = stem_prefix + '-'.join(args) 106 data_filename = output_file_name(options.output_directory, stem, 'h') 107 stdout_filename = output_file_name(options.output_directory, stem, 'out') 108 stderr_filename = output_file_name(options.output_directory, stem, 'err') 109 status_filename = output_file_name(options.output_directory, stem, 'status') 110 shutil.copy(input_file, data_filename) 111 # Pass only the file basename, not the full path, to avoid getting the 112 # directory name in error messages, which would make comparisons 113 # between output directories more difficult. 114 cmd = [os.path.abspath(options.script), 115 '-f', os.path.basename(data_filename)] 116 with open(stdout_filename, 'wb') as out: 117 with open(stderr_filename, 'wb') as err: 118 status = subprocess.call(cmd + args, 119 cwd=options.output_directory, 120 stdin=subprocess.DEVNULL, 121 stdout=out, stderr=err) 122 with open(status_filename, 'w') as status_file: 123 status_file.write('{}\n'.format(status)) 124 return stem + "+", data_filename 125 126 ### A list of symbols to test with. 127 ### This script currently tests what happens when you change a symbol from 128 ### having a value to not having a value or vice versa. This is not 129 ### necessarily useful behavior, and we may not consider it a bug if 130 ### config.py stops handling that case correctly. 131 TEST_SYMBOLS = [ 132 'CUSTOM_SYMBOL', # does not exist 133 'MBEDTLS_AES_C', # set, no value 134 'MBEDTLS_MPI_MAX_SIZE', # unset, has a value 135 'MBEDTLS_NO_UDBL_DIVISION', # unset, in "System support" 136 'MBEDTLS_PLATFORM_ZEROIZE_ALT', # unset, in "Customisation configuration options" 137 ] 138 139 def run_all(options): 140 """Run all the command lines to test.""" 141 presets = list_presets(options) 142 for preset in presets: 143 run_one(options, [preset]) 144 for symbol in TEST_SYMBOLS: 145 run_one(options, ['get', symbol]) 146 (stem, filename) = run_one(options, ['set', symbol]) 147 run_one(options, ['get', symbol], stem_prefix=stem, input_file=filename) 148 run_one(options, ['--force', 'set', symbol]) 149 (stem, filename) = run_one(options, ['set', symbol, 'value']) 150 run_one(options, ['get', symbol], stem_prefix=stem, input_file=filename) 151 run_one(options, ['--force', 'set', symbol, 'value']) 152 run_one(options, ['unset', symbol]) 153 154 def main(): 155 """Command line entry point.""" 156 parser = argparse.ArgumentParser(description=__doc__, 157 formatter_class=argparse.RawDescriptionHelpFormatter) 158 parser.add_argument('-d', metavar='DIR', 159 dest='output_directory', required=True, 160 help="""Output directory.""") 161 parser.add_argument('-f', metavar='FILE', 162 dest='input_file', default='include/mbedtls/mbedtls_config.h', 163 help="""Config file (default: %(default)s).""") 164 parser.add_argument('-p', metavar='PRESET,...', 165 dest='presets', 166 help="""Presets to test (default: guessed from --help).""") 167 parser.add_argument('-s', metavar='FILE', 168 dest='script', default='scripts/config.py', 169 help="""Configuration script (default: %(default)s).""") 170 options = parser.parse_args() 171 prepare_directory(options.output_directory) 172 run_all(options) 173 174 if __name__ == '__main__': 175 main()