analyze_outcomes.py (33142B)
1 #!/usr/bin/env python3 2 3 """Analyze the test outcomes from a full CI run. 4 5 This script can also run on outcomes from a partial run, but the results are 6 less likely to be useful. 7 """ 8 9 import re 10 import typing 11 12 import scripts_path # pylint: disable=unused-import 13 from mbedtls_framework import outcome_analysis 14 15 16 class CoverageTask(outcome_analysis.CoverageTask): 17 """Justify test cases that are never executed.""" 18 19 @staticmethod 20 def _has_word_re(words: typing.Iterable[str], 21 exclude: typing.Optional[str] = None) -> typing.Pattern: 22 """Construct a regex that matches if any of the words appears. 23 24 The occurrence must start and end at a word boundary. 25 26 If exclude is specified, strings containing a match for that 27 regular expression will not match the returned pattern. 28 """ 29 exclude_clause = r'' 30 if exclude: 31 exclude_clause = r'(?!.*' + exclude + ')' 32 return re.compile(exclude_clause + 33 r'.*\b(?:' + r'|'.join(words) + r')\b.*', 34 re.DOTALL) 35 36 IGNORED_TESTS = { 37 'ssl-opt': [ 38 # We don't run ssl-opt.sh with Valgrind on the CI because 39 # it's extremely slow. We don't intend to change this. 40 'DTLS client reconnect from same port: reconnect, nbio, valgrind', 41 # We don't have IPv6 in our CI environment. 42 # https://github.com/Mbed-TLS/mbedtls-test/issues/176 43 'DTLS cookie: enabled, IPv6', 44 # Disabled due to OpenSSL bug. 45 # https://github.com/openssl/openssl/issues/18887 46 'DTLS fragmenting: 3d, openssl client, DTLS 1.2', 47 # We don't run ssl-opt.sh with Valgrind on the CI because 48 # it's extremely slow. We don't intend to change this. 49 'DTLS fragmenting: proxy MTU: auto-reduction (with valgrind)', 50 # It seems that we don't run `ssl-opt.sh` with 51 # `MBEDTLS_USE_PSA_CRYPTO` enabled but `MBEDTLS_SSL_ASYNC_PRIVATE` 52 # disabled. 53 # https://github.com/Mbed-TLS/mbedtls/issues/9581 54 'Opaque key for server authentication: invalid key: decrypt with ECC key, no async', 55 'Opaque key for server authentication: invalid key: ecdh with RSA key, no async', 56 ], 57 'test_suite_config.mbedtls_boolean': [ 58 # We never test with CBC/PKCS5/PKCS12 enabled but 59 # PKCS7 padding disabled. 60 # https://github.com/Mbed-TLS/mbedtls/issues/9580 61 'Config: !MBEDTLS_CIPHER_PADDING_PKCS7', 62 # https://github.com/Mbed-TLS/mbedtls/issues/9583 63 'Config: !MBEDTLS_ECP_NIST_OPTIM', 64 # MBEDTLS_ECP_NO_FALLBACK only affects builds using a partial 65 # alternative implementation of ECP arithmetic (with 66 # MBEDTLS_ECP_INTERNAL_ALT enabled). We don't test those builds. 67 # The configuration enumeration script skips xxx_ALT options 68 # but not MBEDTLS_ECP_NO_FALLBACK, so it appears in the report, 69 # but we don't care about it. 70 'Config: MBEDTLS_ECP_NO_FALLBACK', 71 # Missing coverage of test configurations. 72 # https://github.com/Mbed-TLS/mbedtls/issues/9585 73 'Config: !MBEDTLS_SSL_DTLS_ANTI_REPLAY', 74 # Missing coverage of test configurations. 75 # https://github.com/Mbed-TLS/mbedtls/issues/9585 76 'Config: !MBEDTLS_SSL_DTLS_HELLO_VERIFY', 77 # We don't run test_suite_config when we test this. 78 # https://github.com/Mbed-TLS/mbedtls/issues/9586 79 'Config: !MBEDTLS_SSL_TLS1_3_KEY_EXCHANGE_MODE_PSK_ENABLED', 80 # We only test multithreading with pthreads. 81 # https://github.com/Mbed-TLS/mbedtls/issues/9584 82 'Config: !MBEDTLS_THREADING_PTHREAD', 83 # Built but not tested. 84 # https://github.com/Mbed-TLS/mbedtls/issues/9587 85 'Config: MBEDTLS_AES_USE_HARDWARE_ONLY', 86 # Untested platform-specific optimizations. 87 # https://github.com/Mbed-TLS/mbedtls/issues/9588 88 'Config: MBEDTLS_HAVE_SSE2', 89 # Obsolete configuration option, to be replaced by 90 # PSA entropy drivers. 91 # https://github.com/Mbed-TLS/mbedtls/issues/8150 92 'Config: MBEDTLS_NO_PLATFORM_ENTROPY', 93 # Untested aspect of the platform interface. 94 # https://github.com/Mbed-TLS/mbedtls/issues/9589 95 'Config: MBEDTLS_PLATFORM_NO_STD_FUNCTIONS', 96 # In a client-server build, test_suite_config runs in the 97 # client configuration, so it will never report 98 # MBEDTLS_PSA_CRYPTO_SPM as enabled. That's ok. 99 'Config: MBEDTLS_PSA_CRYPTO_SPM', 100 # We don't test on armv8 yet. 101 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_IF_PRESENT', 102 'Config: MBEDTLS_SHA256_USE_A64_CRYPTO_ONLY', 103 'Config: MBEDTLS_SHA256_USE_ARMV8_A_CRYPTO_ONLY', 104 'Config: MBEDTLS_SHA512_USE_A64_CRYPTO_ONLY', 105 # We don't run test_suite_config when we test this. 106 # https://github.com/Mbed-TLS/mbedtls/issues/9586 107 'Config: MBEDTLS_TEST_CONSTANT_FLOW_VALGRIND', 108 ], 109 'test_suite_config.psa_boolean': [ 110 # We don't test with HMAC disabled. 111 # https://github.com/Mbed-TLS/mbedtls/issues/9591 112 'Config: !PSA_WANT_ALG_HMAC', 113 # The DERIVE key type is always enabled. 114 'Config: !PSA_WANT_KEY_TYPE_DERIVE', 115 # More granularity of key pair type enablement macros 116 # than we care to test. 117 # https://github.com/Mbed-TLS/mbedtls/issues/9590 118 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_EXPORT', 119 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_GENERATE', 120 'Config: !PSA_WANT_KEY_TYPE_DH_KEY_PAIR_IMPORT', 121 # More granularity of key pair type enablement macros 122 # than we care to test. 123 # https://github.com/Mbed-TLS/mbedtls/issues/9590 124 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_EXPORT', 125 'Config: !PSA_WANT_KEY_TYPE_ECC_KEY_PAIR_IMPORT', 126 # We don't test with HMAC disabled. 127 # https://github.com/Mbed-TLS/mbedtls/issues/9591 128 'Config: !PSA_WANT_KEY_TYPE_HMAC', 129 # The PASSWORD key type is always enabled. 130 'Config: !PSA_WANT_KEY_TYPE_PASSWORD', 131 # The PASSWORD_HASH key type is always enabled. 132 'Config: !PSA_WANT_KEY_TYPE_PASSWORD_HASH', 133 # The RAW_DATA key type is always enabled. 134 'Config: !PSA_WANT_KEY_TYPE_RAW_DATA', 135 # More granularity of key pair type enablement macros 136 # than we care to test. 137 # https://github.com/Mbed-TLS/mbedtls/issues/9590 138 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_EXPORT', 139 'Config: !PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_IMPORT', 140 # Algorithm declared but not supported. 141 'Config: PSA_WANT_ALG_CBC_MAC', 142 # Algorithm declared but not supported. 143 'Config: PSA_WANT_ALG_XTS', 144 # Family declared but not supported. 145 'Config: PSA_WANT_ECC_SECP_K1_224', 146 # More granularity of key pair type enablement macros 147 # than we care to test. 148 # https://github.com/Mbed-TLS/mbedtls/issues/9590 149 'Config: PSA_WANT_KEY_TYPE_DH_KEY_PAIR_DERIVE', 150 'Config: PSA_WANT_KEY_TYPE_ECC_KEY_PAIR', 151 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR', 152 'Config: PSA_WANT_KEY_TYPE_RSA_KEY_PAIR_DERIVE', 153 ], 154 'test_suite_config.psa_combinations': [ 155 # We don't test this unusual, but sensible configuration. 156 # https://github.com/Mbed-TLS/mbedtls/issues/9592 157 'Config: PSA_WANT_ALG_DETERMINSTIC_ECDSA without PSA_WANT_ALG_ECDSA', 158 ], 159 'test_suite_pkcs12': [ 160 # We never test with CBC/PKCS5/PKCS12 enabled but 161 # PKCS7 padding disabled. 162 # https://github.com/Mbed-TLS/mbedtls/issues/9580 163 'PBE Decrypt, (Invalid padding & PKCS7 padding disabled)', 164 'PBE Encrypt, pad = 8 (PKCS7 padding disabled)', 165 ], 166 'test_suite_pkcs5': [ 167 # We never test with CBC/PKCS5/PKCS12 enabled but 168 # PKCS7 padding disabled. 169 # https://github.com/Mbed-TLS/mbedtls/issues/9580 170 'PBES2 Decrypt (Invalid padding & PKCS7 padding disabled)', 171 'PBES2 Encrypt, pad=6 (PKCS7 padding disabled)', 172 'PBES2 Encrypt, pad=8 (PKCS7 padding disabled)', 173 ], 174 'test_suite_psa_crypto': [ 175 # We don't test this unusual, but sensible configuration. 176 # https://github.com/Mbed-TLS/mbedtls/issues/9592 177 re.compile(r'.*ECDSA.*only deterministic supported'), 178 ], 179 'test_suite_psa_crypto_metadata': [ 180 # Algorithms declared but not supported. 181 # https://github.com/Mbed-TLS/mbedtls/issues/9579 182 'Asymmetric signature: Ed25519ph', 183 'Asymmetric signature: Ed448ph', 184 'Asymmetric signature: pure EdDSA', 185 'Cipher: XTS', 186 'MAC: CBC_MAC-3DES', 187 'MAC: CBC_MAC-AES-128', 188 'MAC: CBC_MAC-AES-192', 189 'MAC: CBC_MAC-AES-256', 190 ], 191 'test_suite_psa_crypto_not_supported.generated': [ 192 # We never test with DH key support disabled but support 193 # for a DH group enabled. The dependencies of these test 194 # cases don't really make sense. 195 # https://github.com/Mbed-TLS/mbedtls/issues/9574 196 re.compile(r'PSA \w+ DH_.*type not supported'), 197 # We only test partial support for DH with the 2048-bit group 198 # enabled and the other groups disabled. 199 # https://github.com/Mbed-TLS/mbedtls/issues/9575 200 'PSA generate DH_KEY_PAIR(RFC7919) 2048-bit group not supported', 201 'PSA import DH_KEY_PAIR(RFC7919) 2048-bit group not supported', 202 'PSA import DH_PUBLIC_KEY(RFC7919) 2048-bit group not supported', 203 ], 204 'test_suite_psa_crypto_op_fail.generated': [ 205 # We don't test this unusual, but sensible configuration. 206 # https://github.com/Mbed-TLS/mbedtls/issues/9592 207 re.compile(r'.*: !ECDSA but DETERMINISTIC_ECDSA with ECC_.*'), 208 # PBKDF2_HMAC is not in the default configuration, so we don't 209 # enable it in depends.py where we remove hashes. 210 # https://github.com/Mbed-TLS/mbedtls/issues/9576 211 re.compile(r'PSA key_derivation PBKDF2_HMAC\(\w+\): !(?!PBKDF2_HMAC\Z).*'), 212 213 # We never test with the HMAC algorithm enabled but the HMAC 214 # key type disabled. Those dependencies don't really make sense. 215 # https://github.com/Mbed-TLS/mbedtls/issues/9573 216 re.compile(r'.* !HMAC with HMAC'), 217 # There's something wrong with PSA_WANT_ALG_RSA_PSS_ANY_SALT 218 # differing from PSA_WANT_ALG_RSA_PSS. 219 # https://github.com/Mbed-TLS/mbedtls/issues/9578 220 re.compile(r'PSA sign RSA_PSS_ANY_SALT.*!(?:MD|RIPEMD|SHA).*'), 221 # We don't test with ECDH disabled but the key type enabled. 222 # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/161 223 re.compile(r'PSA key_agreement.* !ECDH with ECC_KEY_PAIR\(.*'), 224 # We don't test with FFDH disabled but the key type enabled. 225 # https://github.com/Mbed-TLS/TF-PSA-Crypto/issues/160 226 re.compile(r'PSA key_agreement.* !FFDH with DH_KEY_PAIR\(.*'), 227 ], 228 'test_suite_psa_crypto_op_fail.misc': [ 229 # We don't test this unusual, but sensible configuration. 230 # https://github.com/Mbed-TLS/mbedtls/issues/9592 231 'PSA sign DETERMINISTIC_ECDSA(SHA_256): !ECDSA but DETERMINISTIC_ECDSA with ECC_KEY_PAIR(SECP_R1)', #pylint: disable=line-too-long 232 ], 233 'tls13-misc': [ 234 # Disabled due to OpenSSL bug. 235 # https://github.com/openssl/openssl/issues/10714 236 'TLS 1.3 O->m: resumption', 237 # Disabled due to OpenSSL command line limitation. 238 # https://github.com/Mbed-TLS/mbedtls/issues/9582 239 'TLS 1.3 m->O: resumption with early data', 240 ], 241 } 242 243 244 # The names that we give to classes derived from DriverVSReference do not 245 # follow the usual naming convention, because it's more readable to use 246 # underscores and parts of the configuration names. Also, these classes 247 # are just there to specify some data, so they don't need repetitive 248 # documentation. 249 #pylint: disable=invalid-name,missing-class-docstring 250 251 class DriverVSReference_hash(outcome_analysis.DriverVSReference): 252 REFERENCE = 'test_psa_crypto_config_reference_hash_use_psa' 253 DRIVER = 'test_psa_crypto_config_accel_hash_use_psa' 254 IGNORED_SUITES = [ 255 'shax', 'mdx', # the software implementations that are being excluded 256 'md.psa', # purposefully depends on whether drivers are present 257 'psa_crypto_low_hash.generated', # testing the builtins 258 ] 259 IGNORED_TESTS = { 260 'test_suite_config': [ 261 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), 262 ], 263 'test_suite_platform': [ 264 # Incompatible with sanitizers (e.g. ASan). If the driver 265 # component uses a sanitizer but the reference component 266 # doesn't, we have a PASS vs SKIP mismatch. 267 'Check mbedtls_calloc overallocation', 268 ], 269 } 270 271 class DriverVSReference_hmac(outcome_analysis.DriverVSReference): 272 REFERENCE = 'test_psa_crypto_config_reference_hmac' 273 DRIVER = 'test_psa_crypto_config_accel_hmac' 274 IGNORED_SUITES = [ 275 # These suites require legacy hash support, which is disabled 276 # in the accelerated component. 277 'shax', 'mdx', 278 # This suite tests builtins directly, but these are missing 279 # in the accelerated case. 280 'psa_crypto_low_hash.generated', 281 ] 282 IGNORED_TESTS = { 283 'test_suite_config': [ 284 re.compile(r'.*\bMBEDTLS_(MD5|RIPEMD160|SHA[0-9]+)_.*'), 285 re.compile(r'.*\bMBEDTLS_MD_C\b') 286 ], 287 'test_suite_md': [ 288 # Builtin HMAC is not supported in the accelerate component. 289 re.compile('.*HMAC.*'), 290 # Following tests make use of functions which are not available 291 # when MD_C is disabled, as it happens in the accelerated 292 # test component. 293 re.compile('generic .* Hash file .*'), 294 'MD list', 295 ], 296 'test_suite_md.psa': [ 297 # "legacy only" tests require hash algorithms to be NOT 298 # accelerated, but this of course false for the accelerated 299 # test component. 300 re.compile('PSA dispatch .* legacy only'), 301 ], 302 'test_suite_platform': [ 303 # Incompatible with sanitizers (e.g. ASan). If the driver 304 # component uses a sanitizer but the reference component 305 # doesn't, we have a PASS vs SKIP mismatch. 306 'Check mbedtls_calloc overallocation', 307 ], 308 } 309 310 class DriverVSReference_cipher_aead_cmac(outcome_analysis.DriverVSReference): 311 REFERENCE = 'test_psa_crypto_config_reference_cipher_aead_cmac' 312 DRIVER = 'test_psa_crypto_config_accel_cipher_aead_cmac' 313 # Modules replaced by drivers. 314 IGNORED_SUITES = [ 315 # low-level (block/stream) cipher modules 316 'aes', 'aria', 'camellia', 'des', 'chacha20', 317 # AEAD modes and CMAC 318 'ccm', 'chachapoly', 'cmac', 'gcm', 319 # The Cipher abstraction layer 320 'cipher', 321 ] 322 IGNORED_TESTS = { 323 'test_suite_config': [ 324 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA|CHACHA20|DES)_.*'), 325 re.compile(r'.*\bMBEDTLS_(CCM|CHACHAPOLY|CMAC|GCM)_.*'), 326 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), 327 re.compile(r'.*\bMBEDTLS_CIPHER_.*'), 328 ], 329 # PEM decryption is not supported so far. 330 # The rest of PEM (write, unencrypted read) works though. 331 'test_suite_pem': [ 332 re.compile(r'PEM read .*(AES|DES|\bencrypt).*'), 333 ], 334 'test_suite_platform': [ 335 # Incompatible with sanitizers (e.g. ASan). If the driver 336 # component uses a sanitizer but the reference component 337 # doesn't, we have a PASS vs SKIP mismatch. 338 'Check mbedtls_calloc overallocation', 339 ], 340 # Following tests depend on AES_C/DES_C but are not about 341 # them really, just need to know some error code is there. 342 'test_suite_error': [ 343 'Low and high error', 344 'Single low error' 345 ], 346 # Similar to test_suite_error above. 347 'test_suite_version': [ 348 'Check for MBEDTLS_AES_C when already present', 349 ], 350 # The en/decryption part of PKCS#12 is not supported so far. 351 # The rest of PKCS#12 (key derivation) works though. 352 'test_suite_pkcs12': [ 353 re.compile(r'PBE Encrypt, .*'), 354 re.compile(r'PBE Decrypt, .*'), 355 ], 356 # The en/decryption part of PKCS#5 is not supported so far. 357 # The rest of PKCS#5 (PBKDF2) works though. 358 'test_suite_pkcs5': [ 359 re.compile(r'PBES2 Encrypt, .*'), 360 re.compile(r'PBES2 Decrypt .*'), 361 ], 362 # Encrypted keys are not supported so far. 363 # pylint: disable=line-too-long 364 'test_suite_pkparse': [ 365 'Key ASN1 (Encrypted key PKCS12, trailing garbage data)', 366 'Key ASN1 (Encrypted key PKCS5, trailing garbage data)', 367 re.compile(r'Parse (RSA|EC) Key .*\(.* ([Ee]ncrypted|password).*\)'), 368 ], 369 # Encrypted keys are not supported so far. 370 'ssl-opt': [ 371 'TLS: password protected server key', 372 'TLS: password protected client key', 373 'TLS: password protected server key, two certificates', 374 ], 375 } 376 377 class DriverVSReference_ecp_light_only(outcome_analysis.DriverVSReference): 378 REFERENCE = 'test_psa_crypto_config_reference_ecc_ecp_light_only' 379 DRIVER = 'test_psa_crypto_config_accel_ecc_ecp_light_only' 380 IGNORED_SUITES = [ 381 # Modules replaced by drivers 382 'ecdsa', 'ecdh', 'ecjpake', 383 # Unit tests for the built-in implementation 384 'psa_crypto_ecp', 385 ] 386 IGNORED_TESTS = { 387 'test_suite_config': [ 388 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 389 ], 390 'test_suite_platform': [ 391 # Incompatible with sanitizers (e.g. ASan). If the driver 392 # component uses a sanitizer but the reference component 393 # doesn't, we have a PASS vs SKIP mismatch. 394 'Check mbedtls_calloc overallocation', 395 ], 396 # This test wants a legacy function that takes f_rng, p_rng 397 # arguments, and uses legacy ECDSA for that. The test is 398 # really about the wrapper around the PSA RNG, not ECDSA. 399 'test_suite_random': [ 400 'PSA classic wrapper: ECDSA signature (SECP256R1)', 401 ], 402 # In the accelerated test ECP_C is not set (only ECP_LIGHT is) 403 # so we must ignore disparities in the tests for which ECP_C 404 # is required. 405 'test_suite_ecp': [ 406 re.compile(r'ECP check public-private .*'), 407 re.compile(r'ECP calculate public: .*'), 408 re.compile(r'ECP gen keypair .*'), 409 re.compile(r'ECP point muladd .*'), 410 re.compile(r'ECP point multiplication .*'), 411 re.compile(r'ECP test vectors .*'), 412 ], 413 'test_suite_ssl': [ 414 # This deprecated function is only present when ECP_C is On. 415 'Test configuration of EC groups through mbedtls_ssl_conf_curves()', 416 ], 417 } 418 419 class DriverVSReference_no_ecp_at_all(outcome_analysis.DriverVSReference): 420 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_ecp_at_all' 421 DRIVER = 'test_psa_crypto_config_accel_ecc_no_ecp_at_all' 422 IGNORED_SUITES = [ 423 # Modules replaced by drivers 424 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 425 # Unit tests for the built-in implementation 426 'psa_crypto_ecp', 427 ] 428 IGNORED_TESTS = { 429 'test_suite_config': [ 430 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 431 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 432 ], 433 'test_suite_platform': [ 434 # Incompatible with sanitizers (e.g. ASan). If the driver 435 # component uses a sanitizer but the reference component 436 # doesn't, we have a PASS vs SKIP mismatch. 437 'Check mbedtls_calloc overallocation', 438 ], 439 # See ecp_light_only 440 'test_suite_random': [ 441 'PSA classic wrapper: ECDSA signature (SECP256R1)', 442 ], 443 'test_suite_pkparse': [ 444 # When PK_PARSE_C and ECP_C are defined then PK_PARSE_EC_COMPRESSED 445 # is automatically enabled in build_info.h (backward compatibility) 446 # even if it is disabled in config_psa_crypto_no_ecp_at_all(). As a 447 # consequence compressed points are supported in the reference 448 # component but not in the accelerated one, so they should be skipped 449 # while checking driver's coverage. 450 re.compile(r'Parse EC Key .*compressed\)'), 451 re.compile(r'Parse Public EC Key .*compressed\)'), 452 ], 453 # See ecp_light_only 454 'test_suite_ssl': [ 455 'Test configuration of EC groups through mbedtls_ssl_conf_curves()', 456 ], 457 } 458 459 class DriverVSReference_ecc_no_bignum(outcome_analysis.DriverVSReference): 460 REFERENCE = 'test_psa_crypto_config_reference_ecc_no_bignum' 461 DRIVER = 'test_psa_crypto_config_accel_ecc_no_bignum' 462 IGNORED_SUITES = [ 463 # Modules replaced by drivers 464 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 465 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 466 'bignum.generated', 'bignum.misc', 467 # Unit tests for the built-in implementation 468 'psa_crypto_ecp', 469 ] 470 IGNORED_TESTS = { 471 'test_suite_config': [ 472 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 473 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 474 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 475 ], 476 'test_suite_platform': [ 477 # Incompatible with sanitizers (e.g. ASan). If the driver 478 # component uses a sanitizer but the reference component 479 # doesn't, we have a PASS vs SKIP mismatch. 480 'Check mbedtls_calloc overallocation', 481 ], 482 # See ecp_light_only 483 'test_suite_random': [ 484 'PSA classic wrapper: ECDSA signature (SECP256R1)', 485 ], 486 # See no_ecp_at_all 487 'test_suite_pkparse': [ 488 re.compile(r'Parse EC Key .*compressed\)'), 489 re.compile(r'Parse Public EC Key .*compressed\)'), 490 ], 491 'test_suite_asn1parse': [ 492 'INTEGER too large for mpi', 493 ], 494 'test_suite_asn1write': [ 495 re.compile(r'ASN.1 Write mpi.*'), 496 ], 497 'test_suite_debug': [ 498 re.compile(r'Debug print mbedtls_mpi.*'), 499 ], 500 # See ecp_light_only 501 'test_suite_ssl': [ 502 'Test configuration of EC groups through mbedtls_ssl_conf_curves()', 503 ], 504 } 505 506 class DriverVSReference_ecc_ffdh_no_bignum(outcome_analysis.DriverVSReference): 507 REFERENCE = 'test_psa_crypto_config_reference_ecc_ffdh_no_bignum' 508 DRIVER = 'test_psa_crypto_config_accel_ecc_ffdh_no_bignum' 509 IGNORED_SUITES = [ 510 # Modules replaced by drivers 511 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 'dhm', 512 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 513 'bignum.generated', 'bignum.misc', 514 # Unit tests for the built-in implementation 515 'psa_crypto_ecp', 516 ] 517 IGNORED_TESTS = { 518 'ssl-opt': [ 519 # DHE support in TLS 1.2 requires built-in MBEDTLS_DHM_C 520 # (because it needs custom groups, which PSA does not 521 # provide), even with MBEDTLS_USE_PSA_CRYPTO. 522 re.compile(r'PSK callback:.*\bdhe-psk\b.*'), 523 ], 524 'test_suite_config': [ 525 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 526 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), 527 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECJPAKE|ECP)_.*'), 528 re.compile(r'.*\bMBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED\b.*'), 529 re.compile(r'.*\bMBEDTLS_PK_PARSE_EC_COMPRESSED\b.*'), 530 ], 531 'test_suite_platform': [ 532 # Incompatible with sanitizers (e.g. ASan). If the driver 533 # component uses a sanitizer but the reference component 534 # doesn't, we have a PASS vs SKIP mismatch. 535 'Check mbedtls_calloc overallocation', 536 ], 537 # See ecp_light_only 538 'test_suite_random': [ 539 'PSA classic wrapper: ECDSA signature (SECP256R1)', 540 ], 541 # See no_ecp_at_all 542 'test_suite_pkparse': [ 543 re.compile(r'Parse EC Key .*compressed\)'), 544 re.compile(r'Parse Public EC Key .*compressed\)'), 545 ], 546 'test_suite_asn1parse': [ 547 'INTEGER too large for mpi', 548 ], 549 'test_suite_asn1write': [ 550 re.compile(r'ASN.1 Write mpi.*'), 551 ], 552 'test_suite_debug': [ 553 re.compile(r'Debug print mbedtls_mpi.*'), 554 ], 555 # See ecp_light_only 556 'test_suite_ssl': [ 557 'Test configuration of EC groups through mbedtls_ssl_conf_curves()', 558 ], 559 } 560 561 class DriverVSReference_ffdh_alg(outcome_analysis.DriverVSReference): 562 REFERENCE = 'test_psa_crypto_config_reference_ffdh' 563 DRIVER = 'test_psa_crypto_config_accel_ffdh' 564 IGNORED_SUITES = ['dhm'] 565 IGNORED_TESTS = { 566 'test_suite_config': [ 567 re.compile(r'.*\bMBEDTLS_DHM_C\b.*'), 568 ], 569 'test_suite_platform': [ 570 # Incompatible with sanitizers (e.g. ASan). If the driver 571 # component uses a sanitizer but the reference component 572 # doesn't, we have a PASS vs SKIP mismatch. 573 'Check mbedtls_calloc overallocation', 574 ], 575 } 576 577 class DriverVSReference_tfm_config(outcome_analysis.DriverVSReference): 578 REFERENCE = 'test_tfm_config_no_p256m' 579 DRIVER = 'test_tfm_config_p256m_driver_accel_ec' 580 IGNORED_SUITES = [ 581 # Modules replaced by drivers 582 'asn1parse', 'asn1write', 583 'ecp', 'ecdsa', 'ecdh', 'ecjpake', 584 'bignum_core', 'bignum_random', 'bignum_mod', 'bignum_mod_raw', 585 'bignum.generated', 'bignum.misc', 586 # Unit tests for the built-in implementation 587 'psa_crypto_ecp', 588 ] 589 IGNORED_TESTS = { 590 'test_suite_config': [ 591 re.compile(r'.*\bMBEDTLS_BIGNUM_C\b.*'), 592 re.compile(r'.*\bMBEDTLS_(ASN1\w+)_C\b.*'), 593 re.compile(r'.*\bMBEDTLS_(ECDH|ECDSA|ECP)_.*'), 594 re.compile(r'.*\bMBEDTLS_PSA_P256M_DRIVER_ENABLED\b.*') 595 ], 596 'test_suite_config.crypto_combinations': [ 597 'Config: ECC: Weierstrass curves only', 598 ], 599 'test_suite_platform': [ 600 # Incompatible with sanitizers (e.g. ASan). If the driver 601 # component uses a sanitizer but the reference component 602 # doesn't, we have a PASS vs SKIP mismatch. 603 'Check mbedtls_calloc overallocation', 604 ], 605 # See ecp_light_only 606 'test_suite_random': [ 607 'PSA classic wrapper: ECDSA signature (SECP256R1)', 608 ], 609 } 610 611 class DriverVSReference_rsa(outcome_analysis.DriverVSReference): 612 REFERENCE = 'test_psa_crypto_config_reference_rsa_crypto' 613 DRIVER = 'test_psa_crypto_config_accel_rsa_crypto' 614 IGNORED_SUITES = [ 615 # Modules replaced by drivers. 616 'rsa', 'pkcs1_v15', 'pkcs1_v21', 617 # We temporarily don't care about PK stuff. 618 'pk', 'pkwrite', 'pkparse' 619 ] 620 IGNORED_TESTS = { 621 'test_suite_config': [ 622 re.compile(r'.*\bMBEDTLS_(PKCS1|RSA)_.*'), 623 re.compile(r'.*\bMBEDTLS_GENPRIME\b.*') 624 ], 625 'test_suite_platform': [ 626 # Incompatible with sanitizers (e.g. ASan). If the driver 627 # component uses a sanitizer but the reference component 628 # doesn't, we have a PASS vs SKIP mismatch. 629 'Check mbedtls_calloc overallocation', 630 ], 631 # Following tests depend on RSA_C but are not about 632 # them really, just need to know some error code is there. 633 'test_suite_error': [ 634 'Low and high error', 635 'Single high error' 636 ], 637 # Constant time operations only used for PKCS1_V15 638 'test_suite_constant_time': [ 639 re.compile(r'mbedtls_ct_zeroize_if .*'), 640 re.compile(r'mbedtls_ct_memmove_left .*') 641 ], 642 'test_suite_psa_crypto': [ 643 # We don't support generate_key_custom entry points 644 # in drivers yet. 645 re.compile(r'PSA generate key custom: RSA, e=.*'), 646 re.compile(r'PSA generate key ext: RSA, e=.*'), 647 ], 648 } 649 650 class DriverVSReference_block_cipher_dispatch(outcome_analysis.DriverVSReference): 651 REFERENCE = 'test_full_block_cipher_legacy_dispatch' 652 DRIVER = 'test_full_block_cipher_psa_dispatch' 653 IGNORED_SUITES = [ 654 # Skipped in the accelerated component 655 'aes', 'aria', 'camellia', 656 # These require AES_C, ARIA_C or CAMELLIA_C to be enabled in 657 # order for the cipher module (actually cipher_wrapper) to work 658 # properly. However these symbols are disabled in the accelerated 659 # component so we ignore them. 660 'cipher.ccm', 'cipher.gcm', 'cipher.aes', 'cipher.aria', 661 'cipher.camellia', 662 ] 663 IGNORED_TESTS = { 664 'test_suite_config': [ 665 re.compile(r'.*\bMBEDTLS_(AES|ARIA|CAMELLIA)_.*'), 666 re.compile(r'.*\bMBEDTLS_AES(\w+)_C\b.*'), 667 ], 668 'test_suite_cmac': [ 669 # Following tests require AES_C/ARIA_C/CAMELLIA_C to be enabled, 670 # but these are not available in the accelerated component. 671 'CMAC null arguments', 672 re.compile('CMAC.* (AES|ARIA|Camellia).*'), 673 ], 674 'test_suite_cipher.padding': [ 675 # Following tests require AES_C/CAMELLIA_C to be enabled, 676 # but these are not available in the accelerated component. 677 re.compile('Set( non-existent)? padding with (AES|CAMELLIA).*'), 678 ], 679 'test_suite_pkcs5': [ 680 # The AES part of PKCS#5 PBES2 is not yet supported. 681 # The rest of PKCS#5 (PBKDF2) works, though. 682 re.compile(r'PBES2 .* AES-.*') 683 ], 684 'test_suite_pkparse': [ 685 # PEM (called by pkparse) requires AES_C in order to decrypt 686 # the key, but this is not available in the accelerated 687 # component. 688 re.compile('Parse RSA Key.*(password|AES-).*'), 689 ], 690 'test_suite_pem': [ 691 # Following tests require AES_C, but this is diabled in the 692 # accelerated component. 693 re.compile('PEM read .*AES.*'), 694 'PEM read (unknown encryption algorithm)', 695 ], 696 'test_suite_error': [ 697 # Following tests depend on AES_C but are not about them 698 # really, just need to know some error code is there. 699 'Single low error', 700 'Low and high error', 701 ], 702 'test_suite_version': [ 703 # Similar to test_suite_error above. 704 'Check for MBEDTLS_AES_C when already present', 705 ], 706 'test_suite_platform': [ 707 # Incompatible with sanitizers (e.g. ASan). If the driver 708 # component uses a sanitizer but the reference component 709 # doesn't, we have a PASS vs SKIP mismatch. 710 'Check mbedtls_calloc overallocation', 711 ], 712 } 713 714 #pylint: enable=invalid-name,missing-class-docstring 715 716 717 # List of tasks with a function that can handle this task and additional arguments if required 718 KNOWN_TASKS = { 719 'analyze_coverage': CoverageTask, 720 'analyze_driver_vs_reference_hash': DriverVSReference_hash, 721 'analyze_driver_vs_reference_hmac': DriverVSReference_hmac, 722 'analyze_driver_vs_reference_cipher_aead_cmac': DriverVSReference_cipher_aead_cmac, 723 'analyze_driver_vs_reference_ecp_light_only': DriverVSReference_ecp_light_only, 724 'analyze_driver_vs_reference_no_ecp_at_all': DriverVSReference_no_ecp_at_all, 725 'analyze_driver_vs_reference_ecc_no_bignum': DriverVSReference_ecc_no_bignum, 726 'analyze_driver_vs_reference_ecc_ffdh_no_bignum': DriverVSReference_ecc_ffdh_no_bignum, 727 'analyze_driver_vs_reference_ffdh_alg': DriverVSReference_ffdh_alg, 728 'analyze_driver_vs_reference_tfm_config': DriverVSReference_tfm_config, 729 'analyze_driver_vs_reference_rsa': DriverVSReference_rsa, 730 'analyze_block_cipher_dispatch': DriverVSReference_block_cipher_dispatch, 731 } 732 733 if __name__ == '__main__': 734 outcome_analysis.main(KNOWN_TASKS)