all-core.sh (37498B)
1 # all-core.sh 2 # 3 # Copyright The Mbed TLS Contributors 4 # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 5 6 ################################################################ 7 #### Documentation 8 ################################################################ 9 10 # Purpose 11 # ------- 12 # 13 # To run all tests possible or available on the platform. 14 # 15 # Files structure 16 # --------------- 17 # 18 # The executable entry point for users and the CI is tests/scripts/all.sh. 19 # 20 # The actual content is in the following files: 21 # - all-core.sh contains the core logic for running test components, 22 # processing command line options, reporting results, etc. 23 # - all-helpers.sh contains helper functions used by more than 1 component. 24 # - components-*.sh contain the definitions of the various components. 25 # 26 # The first two parts are shared between repos and branches; 27 # the component files are repo&branch-specific. 28 # 29 # The files all-*.sh and components-*.sh should only define functions and not 30 # run code when sourced; the only exception being that all-core.sh runs 31 # 'shopt' because that is necessary for the rest of the file to parse. 32 # 33 # Notes for users 34 # --------------- 35 # 36 # Warning: the test is destructive. It includes various build modes and 37 # configurations, and can and will arbitrarily change the current CMake 38 # configuration. The following files must be committed into git: 39 # * include/mbedtls/mbedtls_config.h 40 # * Makefile, library/Makefile, programs/Makefile, tests/Makefile, 41 # programs/fuzz/Makefile 42 # After running this script, the CMake cache will be lost and CMake 43 # will no longer be initialised. 44 # 45 # The script assumes the presence of a number of tools: 46 # * Basic Unix tools (Windows users note: a Unix-style find must be before 47 # the Windows find in the PATH) 48 # * Perl 49 # * GNU Make 50 # * CMake 51 # * GCC and Clang (recent enough for using ASan with gcc and MemSan with clang, or valgrind) 52 # * G++ 53 # * arm-gcc and mingw-gcc 54 # * ArmCC 6 (aka armclang), unless invoked with --no-armcc 55 # * OpenSSL and GnuTLS command line tools, in suitable versions for the 56 # interoperability tests. The following are the official versions at the 57 # time of writing: 58 # * GNUTLS_{CLI,SERV} = 3.4.10 59 # * GNUTLS_NEXT_{CLI,SERV} = 3.7.2 60 # * OPENSSL = 1.0.2g (without Debian/Ubuntu patches) 61 # * OPENSSL_NEXT = 3.1.2 62 # See the invocation of check_tools below for details. 63 # 64 # This script must be invoked from the toplevel directory of a git 65 # working copy of Mbed TLS. 66 # 67 # The behavior on an error depends on whether --keep-going (alias -k) 68 # is in effect. 69 # * Without --keep-going: the script stops on the first error without 70 # cleaning up. This lets you work in the configuration of the failing 71 # component. 72 # * With --keep-going: the script runs all requested components and 73 # reports failures at the end. In particular the script always cleans 74 # up on exit. 75 # 76 # Note that the output is not saved. You may want to run 77 # script -c tests/scripts/all.sh 78 # or 79 # tests/scripts/all.sh >all.log 2>&1 80 # 81 # Notes for maintainers 82 # --------------------- 83 # 84 # The bulk of the code is organized into functions that follow one of the 85 # following naming conventions: 86 # * in all-core.sh: 87 # * pre_XXX: things to do before running the tests, in order. 88 # * post_XXX: things to do after running the tests. 89 # * in components-*.sh: 90 # * component_XXX: independent components. They can be run in any order. 91 # * component_check_XXX: quick tests that aren't worth parallelizing. 92 # * component_build_XXX: build things but don't run them. 93 # * component_test_XXX: build and test. 94 # * component_release_XXX: tests that the CI should skip during PR testing. 95 # * support_XXX: if support_XXX exists and returns false then 96 # component_XXX is not run by default. 97 # * in various files: 98 # * other: miscellaneous support functions. 99 # 100 # Each component must start by invoking `msg` with a short informative message. 101 # 102 # Warning: due to the way bash detects errors, the failure of a command 103 # inside 'if' or '!' is not detected. Use the 'not' function instead of '!'. 104 # 105 # Each component is executed in a separate shell process. The component 106 # fails if any command in it returns a non-zero status. 107 # 108 # The framework performs some cleanup tasks after each component. This 109 # means that components can assume that the working directory is in a 110 # cleaned-up state, and don't need to perform the cleanup themselves. 111 # * Run `make clean`. 112 # * Restore the various config files (potentially modified by config.py) from 113 # a backup made when starting the script. 114 # * If in Mbed TLS, restore the various `Makefile`s (potentially modified by 115 # in-tree use of CMake) from a backup made when starting the script. (Note: 116 # if the files look generated when starting the script, they will be 117 # restored from the git index before making the backup.) 118 119 120 ################################################################ 121 #### Initialization and command line parsing 122 ################################################################ 123 124 # Enable ksh/bash extended file matching patterns. 125 # Must come before function definitions or some of them wouldn't parse. 126 shopt -s extglob 127 128 pre_set_shell_options () { 129 # Abort on errors (even on the left-hand side of a pipe). 130 # Treat uninitialised variables as errors. 131 set -e -o pipefail -u 132 } 133 134 pre_check_environment () { 135 136 source $FRAMEWORK/scripts/project_detection.sh 137 138 if in_mbedtls_repo || in_tf_psa_crypto_repo; then :; else 139 echo "Must be run from Mbed TLS / TF-PSA-Crypto root" >&2 140 exit 1 141 fi 142 } 143 144 # Must be called before pre_initialize_variables which sets ALL_COMPONENTS. 145 pre_load_components () { 146 # Include the components from components.sh 147 # Use a path relative to the current directory, aka project's root. 148 for file in tests/scripts/components-*.sh; do 149 source $file 150 done 151 } 152 153 pre_initialize_variables () { 154 if in_mbedtls_repo; then 155 CONFIG_H='include/mbedtls/mbedtls_config.h' 156 if in_3_6_branch; then 157 CRYPTO_CONFIG_H='include/psa/crypto_config.h' 158 # helper_armc6_build_test() relies on these being defined, 159 # but empty if the paths don't exist (as in 3.6). 160 PSA_CORE_PATH='' 161 BUILTIN_SRC_PATH='' 162 CONFIG_TEST_DRIVER_H='tests/configs/config_test_driver.h' 163 else 164 CRYPTO_CONFIG_H='tf-psa-crypto/include/psa/crypto_config.h' 165 PSA_CORE_PATH='tf-psa-crypto/core' 166 BUILTIN_SRC_PATH='tf-psa-crypto/drivers/builtin/src' 167 CONFIG_TEST_DRIVER_H='tf-psa-crypto/tests/configs/crypto_config_test_driver.h' 168 MBEDTLS_ROOT_DIR="$PWD" 169 TF_PSA_CRYPTO_ROOT_DIR="$PWD/tf-psa-crypto" 170 fi 171 config_files="$CONFIG_H $CRYPTO_CONFIG_H $CONFIG_TEST_DRIVER_H" 172 else 173 CRYPTO_CONFIG_H='include/psa/crypto_config.h' 174 PSA_CORE_PATH='core' 175 BUILTIN_SRC_PATH='drivers/builtin/src' 176 CONFIG_TEST_DRIVER_H='tests/configs/config_test_driver.h' 177 TF_PSA_CRYPTO_ROOT_DIR="$PWD" 178 MBEDTLS_ROOT_DIR="" 179 180 config_files="$CRYPTO_CONFIG_H $CONFIG_TEST_DRIVER_H" 181 fi 182 183 # Files that are clobbered by some jobs will be backed up. Use a different 184 # suffix from auxiliary scripts so that all.sh and auxiliary scripts can 185 # independently decide when to remove the backup file. 186 backup_suffix='.all.bak' 187 # Files clobbered by config.py 188 files_to_back_up="$config_files" 189 if in_mbedtls_repo; then 190 # Files clobbered by in-tree cmake 191 files_to_back_up="$files_to_back_up Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile" 192 fi 193 194 append_outcome=0 195 MEMORY=0 196 FORCE=0 197 QUIET=0 198 KEEP_GOING=0 199 200 # Seed value used with the --release-test option. 201 # 202 # See also RELEASE_SEED in basic-build-test.sh. Debugging is easier if 203 # both values are kept in sync. If you change the value here because it 204 # breaks some tests, you'll definitely want to change it in 205 # basic-build-test.sh as well. 206 RELEASE_SEED=1 207 208 # Specify character collation for regular expressions and sorting with C locale 209 export LC_COLLATE=C 210 211 : ${MBEDTLS_TEST_OUTCOME_FILE=} 212 : ${MBEDTLS_TEST_PLATFORM="$(uname -s | tr -c \\n0-9A-Za-z _)-$(uname -m | tr -c \\n0-9A-Za-z _)"} 213 export MBEDTLS_TEST_OUTCOME_FILE 214 export MBEDTLS_TEST_PLATFORM 215 216 # Default commands, can be overridden by the environment 217 : ${OPENSSL:="openssl"} 218 : ${OPENSSL_NEXT:="$OPENSSL"} 219 : ${GNUTLS_CLI:="gnutls-cli"} 220 : ${GNUTLS_SERV:="gnutls-serv"} 221 : ${OUT_OF_SOURCE_DIR:=$PWD/out_of_source_build} 222 : ${ARMC6_BIN_DIR:=/usr/bin} 223 : ${ARM_NONE_EABI_GCC_PREFIX:=arm-none-eabi-} 224 : ${ARM_LINUX_GNUEABI_GCC_PREFIX:=arm-linux-gnueabi-} 225 : ${ARM_LINUX_GNUEABIHF_GCC_PREFIX:=arm-linux-gnueabihf-} 226 : ${AARCH64_LINUX_GNU_GCC_PREFIX:=aarch64-linux-gnu-} 227 : ${CLANG_LATEST:="clang-latest"} 228 : ${CLANG_EARLIEST:="clang-earliest"} 229 : ${GCC_LATEST:="gcc-latest"} 230 : ${GCC_EARLIEST:="gcc-earliest"} 231 # if MAKEFLAGS is not set add the -j option to speed up invocations of make 232 if [ -z "${MAKEFLAGS+set}" ]; then 233 export MAKEFLAGS="-j$(all_sh_nproc)" 234 fi 235 # if CC is not set, use clang by default (if present) to improve build times 236 if [ -z "${CC+set}" ] && (type clang > /dev/null 2>&1); then 237 export CC="clang" 238 fi 239 240 if [ -n "${OPENSSL_3+set}" ]; then 241 export OPENSSL_NEXT="$OPENSSL_3" 242 fi 243 244 # Include more verbose output for failing tests run by CMake or make 245 export CTEST_OUTPUT_ON_FAILURE=1 246 247 # CFLAGS and LDFLAGS for Asan builds that don't use CMake 248 # default to -O2, use -Ox _after_ this if you want another level 249 ASAN_CFLAGS='-O2 -Werror -fsanitize=address,undefined -fno-sanitize-recover=all' 250 # Normally, tests should use this compiler for ASAN testing 251 ASAN_CC=clang 252 253 # Platform tests have an allocation that returns null 254 export ASAN_OPTIONS="allocator_may_return_null=1" 255 export MSAN_OPTIONS="allocator_may_return_null=1" 256 257 # Gather the list of available components. These are the functions 258 # defined in this script whose name starts with "component_". 259 ALL_COMPONENTS=$(compgen -A function component_ | sed 's/component_//') 260 261 PSASIM_PATH='tests/psa-client-server/psasim/' 262 263 # Delay determining SUPPORTED_COMPONENTS until the command line options have a chance to override 264 # the commands set by the environment 265 } 266 267 setup_quiet_wrappers() 268 { 269 # Pick up "quiet" wrappers for make and cmake, which don't output very much 270 # unless there is an error. This reduces logging overhead in the CI. 271 # 272 # Note that the cmake wrapper breaks unless we use an absolute path here. 273 if [[ -e ${PWD}/framework/scripts/quiet ]]; then 274 export PATH=${PWD}/framework/scripts/quiet:$PATH 275 fi 276 } 277 278 # Test whether the component $1 is included in the command line patterns. 279 is_component_included() 280 { 281 # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS 282 # only does word splitting. 283 set -f 284 for pattern in $COMMAND_LINE_COMPONENTS; do 285 set +f 286 case ${1#component_} in $pattern) return 0;; esac 287 done 288 set +f 289 return 1 290 } 291 292 usage() 293 { 294 cat <<EOF 295 Usage: $0 [OPTION]... [COMPONENT]... 296 Run mbedtls release validation tests. 297 By default, run all tests. With one or more COMPONENT, run only those. 298 COMPONENT can be the name of a component or a shell wildcard pattern. 299 300 Examples: 301 $0 "check_*" 302 Run all sanity checks. 303 $0 --no-armcc --except test_memsan 304 Run everything except builds that require armcc and MemSan. 305 306 Special options: 307 -h|--help Print this help and exit. 308 --list-all-components List all available test components and exit. 309 --list-components List components supported on this platform and exit. 310 311 General options: 312 -q|--quiet Only output component names, and errors if any. 313 -f|--force Force the tests to overwrite any modified files. 314 -k|--keep-going Run all tests and report errors at the end. 315 -m|--memory Additional optional memory tests. 316 --append-outcome Append to the outcome file (if used). 317 --arm-none-eabi-gcc-prefix=<string> 318 Prefix for a cross-compiler for arm-none-eabi 319 (default: "${ARM_NONE_EABI_GCC_PREFIX}") 320 --arm-linux-gnueabi-gcc-prefix=<string> 321 Prefix for a cross-compiler for arm-linux-gnueabi 322 (default: "${ARM_LINUX_GNUEABI_GCC_PREFIX}") 323 --arm-linux-gnueabihf-gcc-prefix=<string> 324 Prefix for a cross-compiler for arm-linux-gnueabihf 325 (default: "${ARM_LINUX_GNUEABIHF_GCC_PREFIX}") 326 --aarch64-linux-gnu-gcc-prefix=<string> 327 Prefix for a cross-compiler for aarch64-linux-gnu 328 (default: "${AARCH64_LINUX_GNU_GCC_PREFIX}") 329 --armcc Run ARM Compiler builds (on by default). 330 --restore First clean up the build tree, restoring backed up 331 files. Do not run any components unless they are 332 explicitly specified. 333 --error-test Error test mode: run a failing function in addition 334 to any specified component. May be repeated. 335 --except Exclude the COMPONENTs listed on the command line, 336 instead of running only those. 337 --no-append-outcome Write a new outcome file and analyze it (default). 338 --no-armcc Skip ARM Compiler builds. 339 --no-force Refuse to overwrite modified files (default). 340 --no-keep-going Stop at the first error (default). 341 --no-memory No additional memory tests (default). 342 --no-quiet Print full output from components. 343 --out-of-source-dir=<path> Directory used for CMake out-of-source build tests. 344 --outcome-file=<path> File where test outcomes are written (not done if 345 empty; default: \$MBEDTLS_TEST_OUTCOME_FILE). 346 --random-seed Use a random seed value for randomized tests (default). 347 -r|--release-test Run this script in release mode. This fixes the seed value to ${RELEASE_SEED}. 348 -s|--seed Integer seed value to use for this test run. 349 350 Tool path options: 351 --armc6-bin-dir=<ARMC6_bin_dir_path> ARM Compiler 6 bin directory. 352 --clang-earliest=<Clang_earliest_path> Earliest version of clang available 353 --clang-latest=<Clang_latest_path> Latest version of clang available 354 --gcc-earliest=<GCC_earliest_path> Earliest version of GCC available 355 --gcc-latest=<GCC_latest_path> Latest version of GCC available 356 --gnutls-cli=<GnuTLS_cli_path> GnuTLS client executable to use for most tests. 357 --gnutls-serv=<GnuTLS_serv_path> GnuTLS server executable to use for most tests. 358 --openssl=<OpenSSL_path> OpenSSL executable to use for most tests. 359 --openssl-next=<OpenSSL_path> OpenSSL executable to use for recent things like ARIA 360 EOF 361 } 362 363 # Cleanup before/after running a component. 364 # Remove built files as well as the cmake cache/config. 365 # Does not remove generated source files. 366 cleanup() 367 { 368 if in_mbedtls_repo; then 369 command make clean 370 fi 371 372 # Remove CMake artefacts 373 find . -name .git -prune -o \ 374 -iname CMakeFiles -exec rm -rf {} \+ -o \ 375 \( -iname cmake_install.cmake -o \ 376 -iname CTestTestfile.cmake -o \ 377 -iname CMakeCache.txt -o \ 378 -path './cmake/*.cmake' \) -exec rm -f {} \+ 379 # Remove Makefiles generated by in-tree CMake builds 380 # (Not all files will exist in all branches, but that's OK.) 381 rm -f 3rdparty/Makefile 3rdparty/*/Makefile 382 rm -f pkgconfig/Makefile framework/Makefile 383 rm -f include/Makefile programs/!(fuzz)/Makefile 384 rm -f tf-psa-crypto/Makefile tf-psa-crypto/include/Makefile 385 rm -f tf-psa-crypto/core/Makefile tf-psa-crypto/drivers/Makefile 386 rm -f tf-psa-crypto/tests/Makefile 387 rm -f tf-psa-crypto/drivers/everest/Makefile 388 rm -f tf-psa-crypto/drivers/p256-m/Makefile 389 rm -f tf-psa-crypto/drivers/builtin/Makefile 390 rm -f tf-psa-crypto/drivers/builtin/src/Makefile 391 392 # Remove any artifacts from the component_test_cmake_as_subdirectory test. 393 rm -rf programs/test/cmake_subproject/build 394 rm -f programs/test/cmake_subproject/Makefile 395 rm -f programs/test/cmake_subproject/cmake_subproject 396 397 # Remove any artifacts from the component_test_cmake_as_package test. 398 rm -rf programs/test/cmake_package/build 399 rm -f programs/test/cmake_package/Makefile 400 rm -f programs/test/cmake_package/cmake_package 401 402 # Remove any artifacts from the component_test_cmake_as_installed_package test. 403 rm -rf programs/test/cmake_package_install/build 404 rm -f programs/test/cmake_package_install/Makefile 405 rm -f programs/test/cmake_package_install/cmake_package_install 406 407 # Remove out of source directory 408 if in_tf_psa_crypto_repo; then 409 rm -rf "$OUT_OF_SOURCE_DIR" 410 fi 411 412 # Restore files that may have been clobbered by the job 413 restore_backed_up_files 414 } 415 416 # Restore files that may have been clobbered 417 restore_backed_up_files () { 418 for x in $files_to_back_up; do 419 if [[ -e "$x$backup_suffix" ]]; then 420 cp -p "$x$backup_suffix" "$x" 421 fi 422 done 423 } 424 425 # Final cleanup when this script exits (except when exiting on a failure 426 # in non-keep-going mode). 427 final_cleanup () { 428 cleanup 429 430 for x in $files_to_back_up; do 431 rm -f "$x$backup_suffix" 432 done 433 } 434 435 # Executed on exit. May be redefined depending on command line options. 436 final_report () { 437 : 438 } 439 440 fatal_signal () { 441 final_cleanup 442 final_report $1 443 trap - $1 444 kill -$1 $$ 445 } 446 447 pre_set_signal_handlers () { 448 trap 'fatal_signal HUP' HUP 449 trap 'fatal_signal INT' INT 450 trap 'fatal_signal TERM' TERM 451 } 452 453 # Number of processors on this machine. Used as the default setting 454 # for parallel make. 455 all_sh_nproc () 456 { 457 { 458 nproc || # Linux 459 sysctl -n hw.ncpuonline || # NetBSD, OpenBSD 460 sysctl -n hw.ncpu || # FreeBSD 461 echo 1 462 } 2>/dev/null 463 } 464 465 msg() 466 { 467 if [ -n "${current_component:-}" ]; then 468 current_section="${current_component#component_}: $1" 469 else 470 current_section="$1" 471 fi 472 473 if [ $QUIET -eq 1 ]; then 474 return 475 fi 476 477 echo "" 478 echo "******************************************************************" 479 echo "* $current_section " 480 printf "* "; date 481 echo "******************************************************************" 482 } 483 484 err_msg() 485 { 486 echo "$1" >&2 487 } 488 489 check_tools() 490 { 491 for tool in "$@"; do 492 if ! `type "$tool" >/dev/null 2>&1`; then 493 err_msg "$tool not found!" 494 exit 1 495 fi 496 done 497 } 498 499 pre_parse_command_line () { 500 COMMAND_LINE_COMPONENTS= 501 all_except=0 502 error_test=0 503 list_components=0 504 restore_first=0 505 no_armcc= 506 507 # Note that legacy options are ignored instead of being omitted from this 508 # list of options, so invocations that worked with previous version of 509 # all.sh will still run and work properly. 510 while [ $# -gt 0 ]; do 511 case "$1" in 512 --append-outcome) append_outcome=1;; 513 --arm-none-eabi-gcc-prefix) shift; ARM_NONE_EABI_GCC_PREFIX="$1";; 514 --arm-linux-gnueabi-gcc-prefix) shift; ARM_LINUX_GNUEABI_GCC_PREFIX="$1";; 515 --arm-linux-gnueabihf-gcc-prefix) shift; ARM_LINUX_GNUEABIHF_GCC_PREFIX="$1";; 516 --aarch64-linux-gnu-gcc-prefix) shift; AARCH64_LINUX_GNU_GCC_PREFIX="$1";; 517 --armcc) no_armcc=;; 518 --armc6-bin-dir) shift; ARMC6_BIN_DIR="$1";; 519 --clang-earliest) shift; CLANG_EARLIEST="$1";; 520 --clang-latest) shift; CLANG_LATEST="$1";; 521 --error-test) error_test=$((error_test + 1));; 522 --except) all_except=1;; 523 --force|-f) FORCE=1;; 524 --gcc-earliest) shift; GCC_EARLIEST="$1";; 525 --gcc-latest) shift; GCC_LATEST="$1";; 526 --gnutls-cli) shift; GNUTLS_CLI="$1";; 527 --gnutls-legacy-cli) shift;; # ignored for backward compatibility 528 --gnutls-legacy-serv) shift;; # ignored for backward compatibility 529 --gnutls-serv) shift; GNUTLS_SERV="$1";; 530 --help|-h) usage; exit;; 531 --keep-going|-k) KEEP_GOING=1;; 532 --list-all-components) printf '%s\n' $ALL_COMPONENTS; exit;; 533 --list-components) list_components=1;; 534 --memory|-m) MEMORY=1;; 535 --no-append-outcome) append_outcome=0;; 536 --no-armcc) no_armcc=1;; 537 --no-force) FORCE=0;; 538 --no-keep-going) KEEP_GOING=0;; 539 --no-memory) MEMORY=0;; 540 --no-quiet) QUIET=0;; 541 --openssl) shift; OPENSSL="$1";; 542 --openssl-next) shift; OPENSSL_NEXT="$1";; 543 --outcome-file) shift; MBEDTLS_TEST_OUTCOME_FILE="$1";; 544 --out-of-source-dir) shift; OUT_OF_SOURCE_DIR="$1";; 545 --quiet|-q) QUIET=1;; 546 --random-seed) unset SEED;; 547 --release-test|-r) SEED=$RELEASE_SEED;; 548 --restore) restore_first=1;; 549 --seed|-s) shift; SEED="$1";; 550 -*) 551 echo >&2 "Unknown option: $1" 552 echo >&2 "Run $0 --help for usage." 553 exit 120 554 ;; 555 *) COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS $1";; 556 esac 557 shift 558 done 559 560 # Exclude components that are not supported on this platform. 561 SUPPORTED_COMPONENTS= 562 for component in $ALL_COMPONENTS; do 563 case $(type "support_$component" 2>&1) in 564 *' function'*) 565 if ! support_$component; then continue; fi;; 566 esac 567 SUPPORTED_COMPONENTS="$SUPPORTED_COMPONENTS $component" 568 done 569 570 if [ $list_components -eq 1 ]; then 571 printf '%s\n' $SUPPORTED_COMPONENTS 572 exit 573 fi 574 575 # With no list of components, run everything. 576 if [ -z "$COMMAND_LINE_COMPONENTS" ] && [ $restore_first -eq 0 ]; then 577 all_except=1 578 fi 579 580 # --no-armcc is a legacy option. The modern way is --except '*_armcc*'. 581 # Ignore it if components are listed explicitly on the command line. 582 if [ -n "$no_armcc" ] && [ $all_except -eq 1 ]; then 583 COMMAND_LINE_COMPONENTS="$COMMAND_LINE_COMPONENTS *_armcc*" 584 fi 585 586 # Error out if an explicitly requested component doesn't exist. 587 if [ $all_except -eq 0 ]; then 588 unsupported=0 589 # Temporarily disable wildcard expansion so that $COMMAND_LINE_COMPONENTS 590 # only does word splitting. 591 set -f 592 for component in $COMMAND_LINE_COMPONENTS; do 593 set +f 594 # If the requested name includes a wildcard character, don't 595 # check it. Accept wildcard patterns that don't match anything. 596 case $component in 597 *[*?\[]*) continue;; 598 esac 599 case " $SUPPORTED_COMPONENTS " in 600 *" $component "*) :;; 601 *) 602 echo >&2 "Component $component was explicitly requested, but is not known or not supported." 603 unsupported=$((unsupported + 1));; 604 esac 605 done 606 set +f 607 if [ $unsupported -ne 0 ]; then 608 exit 2 609 fi 610 fi 611 612 # Build the list of components to run. 613 RUN_COMPONENTS= 614 for component in $SUPPORTED_COMPONENTS; do 615 if is_component_included "$component"; [ $? -eq $all_except ]; then 616 RUN_COMPONENTS="$RUN_COMPONENTS $component" 617 fi 618 done 619 620 unset all_except 621 unset no_armcc 622 } 623 624 pre_check_git () { 625 if [ $FORCE -eq 1 ]; then 626 rm -rf "$OUT_OF_SOURCE_DIR" 627 git checkout-index -f -q $config_files 628 cleanup 629 else 630 631 if [ -d "$OUT_OF_SOURCE_DIR" ]; then 632 echo "Warning - there is an existing directory at '$OUT_OF_SOURCE_DIR'" >&2 633 echo "You can either delete this directory manually, or force the test by rerunning" 634 echo "the script as: $0 --force --out-of-source-dir $OUT_OF_SOURCE_DIR" 635 exit 1 636 fi 637 638 for config in $config_files; do 639 if ! git diff --quiet "$config"; then 640 err_msg "Warning - the configuration file '$config' has been edited. " 641 echo "You can either delete or preserve your work, or force the test by rerunning the" 642 echo "script as: $0 --force" 643 exit 1 644 fi 645 done 646 fi 647 } 648 649 pre_restore_files () { 650 # If the makefiles have been generated by a framework such as cmake, 651 # restore them from git. If the makefiles look like modifications from 652 # the ones checked into git, take care not to modify them. Whatever 653 # this function leaves behind is what the script will restore before 654 # each component. 655 case "$(head -n1 Makefile)" in 656 *[Gg]enerated*) 657 git update-index --no-skip-worktree Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile 658 git checkout -- Makefile library/Makefile programs/Makefile tests/Makefile programs/fuzz/Makefile 659 ;; 660 esac 661 } 662 663 pre_back_up () { 664 for x in $files_to_back_up; do 665 cp -p "$x" "$x$backup_suffix" 666 done 667 } 668 669 pre_setup_keep_going () { 670 failure_count=0 # Number of failed components 671 last_failure_status=0 # Last failure status in this component 672 673 # See err_trap 674 previous_failure_status=0 675 previous_failed_command= 676 previous_failure_funcall_depth=0 677 unset report_failed_command 678 679 start_red= 680 end_color= 681 if [ -t 1 ]; then 682 case "${TERM:-}" in 683 *color*|cygwin|linux|rxvt*|screen|[Eex]term*) 684 start_red=$(printf '\033[31m') 685 end_color=$(printf '\033[0m') 686 ;; 687 esac 688 fi 689 690 # Keep a summary of failures in a file. We'll print it out at the end. 691 failure_summary_file=$PWD/all-sh-failures-$$.log 692 : >"$failure_summary_file" 693 694 # Whether it makes sense to keep a component going after the specified 695 # command fails (test command) or not (configure or build). 696 # This function normally receives the failing simple command 697 # ($BASH_COMMAND) as an argument, but if $report_failed_command is set, 698 # this is passed instead. 699 # This doesn't have to be 100% accurate: all failures are recorded anyway. 700 # False positives result in running things that can't be expected to 701 # work. False negatives result in things not running after something else 702 # failed even though they might have given useful feedback. 703 can_keep_going_after_failure () { 704 case "$1" in 705 "msg "*) false;; 706 "cd "*) false;; 707 "diff "*) true;; 708 *make*[\ /]tests*) false;; # make tests, make CFLAGS=-I../tests, ... 709 *test*) true;; # make test, tests/stuff, env V=v tests/stuff, ... 710 *make*check*) true;; 711 "grep "*) true;; 712 "[ "*) true;; 713 "! "*) true;; 714 *) false;; 715 esac 716 } 717 718 # This function runs if there is any error in a component. 719 # It must either exit with a nonzero status, or set 720 # last_failure_status to a nonzero value. 721 err_trap () { 722 # Save $? (status of the failing command). This must be the very 723 # first thing, before $? is overridden. 724 last_failure_status=$? 725 failed_command=${report_failed_command-$BASH_COMMAND} 726 727 if [[ $last_failure_status -eq $previous_failure_status && 728 "$failed_command" == "$previous_failed_command" && 729 ${#FUNCNAME[@]} == $((previous_failure_funcall_depth - 1)) ]] 730 then 731 # The same command failed twice in a row, but this time one level 732 # less deep in the function call stack. This happens when the last 733 # command of a function returns a nonzero status, and the function 734 # returns that same status. Ignore the second failure. 735 previous_failure_funcall_depth=${#FUNCNAME[@]} 736 return 737 fi 738 previous_failure_status=$last_failure_status 739 previous_failed_command=$failed_command 740 previous_failure_funcall_depth=${#FUNCNAME[@]} 741 742 text="$current_section: $failed_command -> $last_failure_status" 743 echo "${start_red}^^^^$text^^^^${end_color}" >&2 744 echo "$text" >>"$failure_summary_file" 745 746 # If the command is fatal (configure or build command), stop this 747 # component. Otherwise (test command) keep the component running 748 # (run more tests from the same build). 749 if ! can_keep_going_after_failure "$failed_command"; then 750 exit $last_failure_status 751 fi 752 } 753 754 final_report () { 755 if [ $failure_count -gt 0 ]; then 756 echo 757 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" 758 echo "${start_red}FAILED: $failure_count components${end_color}" 759 cat "$failure_summary_file" 760 echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" 761 elif [ -z "${1-}" ]; then 762 echo "SUCCESS :)" 763 fi 764 if [ -n "${1-}" ]; then 765 echo "Killed by SIG$1." 766 fi 767 rm -f "$failure_summary_file" 768 if [ $failure_count -gt 0 ]; then 769 exit 1 770 fi 771 } 772 } 773 774 # '! true' does not trigger the ERR trap. Arrange to trigger it, with 775 # a reasonably informative error message (not just "$@"). 776 not () { 777 if "$@"; then 778 report_failed_command="! $*" 779 false 780 unset report_failed_command 781 fi 782 } 783 784 pre_prepare_outcome_file () { 785 case "$MBEDTLS_TEST_OUTCOME_FILE" in 786 [!/]*) MBEDTLS_TEST_OUTCOME_FILE="$PWD/$MBEDTLS_TEST_OUTCOME_FILE";; 787 esac 788 if [ -n "$MBEDTLS_TEST_OUTCOME_FILE" ] && [ "$append_outcome" -eq 0 ]; then 789 rm -f "$MBEDTLS_TEST_OUTCOME_FILE" 790 fi 791 } 792 793 pre_print_configuration () { 794 if [ $QUIET -eq 1 ]; then 795 return 796 fi 797 798 msg "info: $0 configuration" 799 echo "MEMORY: $MEMORY" 800 echo "FORCE: $FORCE" 801 echo "MBEDTLS_TEST_OUTCOME_FILE: ${MBEDTLS_TEST_OUTCOME_FILE:-(none)}" 802 echo "SEED: ${SEED-"UNSET"}" 803 echo 804 echo "OPENSSL: $OPENSSL" 805 echo "OPENSSL_NEXT: $OPENSSL_NEXT" 806 echo "GNUTLS_CLI: $GNUTLS_CLI" 807 echo "GNUTLS_SERV: $GNUTLS_SERV" 808 echo "ARMC6_BIN_DIR: $ARMC6_BIN_DIR" 809 } 810 811 # Make sure the tools we need are available. 812 pre_check_tools () { 813 # Build the list of variables to pass to output_env.sh. 814 set env 815 816 case " $RUN_COMPONENTS " in 817 # Require OpenSSL and GnuTLS if running any tests (as opposed to 818 # only doing builds). Not all tests run OpenSSL and GnuTLS, but this 819 # is a good enough approximation in practice. 820 *" test_"* | *" release_test_"*) 821 # To avoid setting OpenSSL and GnuTLS for each call to compat.sh 822 # and ssl-opt.sh, we just export the variables they require. 823 export OPENSSL="$OPENSSL" 824 export GNUTLS_CLI="$GNUTLS_CLI" 825 export GNUTLS_SERV="$GNUTLS_SERV" 826 # Avoid passing --seed flag in every call to ssl-opt.sh 827 if [ -n "${SEED-}" ]; then 828 export SEED 829 fi 830 set "$@" OPENSSL="$OPENSSL" 831 set "$@" GNUTLS_CLI="$GNUTLS_CLI" GNUTLS_SERV="$GNUTLS_SERV" 832 check_tools "$OPENSSL" "$OPENSSL_NEXT" \ 833 "$GNUTLS_CLI" "$GNUTLS_SERV" 834 ;; 835 esac 836 837 case " $RUN_COMPONENTS " in 838 *_doxygen[_\ ]*) check_tools "doxygen" "dot";; 839 esac 840 841 case " $RUN_COMPONENTS " in 842 *_arm_none_eabi_gcc[_\ ]*) check_tools "${ARM_NONE_EABI_GCC_PREFIX}gcc";; 843 esac 844 845 case " $RUN_COMPONENTS " in 846 *_mingw[_\ ]*) check_tools "i686-w64-mingw32-gcc";; 847 esac 848 849 case " $RUN_COMPONENTS " in 850 *" test_zeroize "*) check_tools "gdb";; 851 esac 852 853 case " $RUN_COMPONENTS " in 854 *_armcc*) 855 ARMC6_CC="$ARMC6_BIN_DIR/armclang" 856 ARMC6_LINK="$ARMC6_BIN_DIR/armlink" 857 ARMC6_AR="$ARMC6_BIN_DIR/armar" 858 ARMC6_FROMELF="$ARMC6_BIN_DIR/fromelf" 859 check_tools "$ARMC6_CC" "$ARMC6_AR" "$ARMC6_FROMELF";; 860 esac 861 862 # past this point, no call to check_tool, only printing output 863 if [ $QUIET -eq 1 ]; then 864 return 865 fi 866 867 msg "info: output_env.sh" 868 case $RUN_COMPONENTS in 869 *_armcc*) 870 set "$@" ARMC6_CC="$ARMC6_CC" RUN_ARMCC=1;; 871 *) set "$@" RUN_ARMCC=0;; 872 esac 873 # Use a path relative to the currently-sourced file. 874 "$@" "${BASH_SOURCE%/*}"/output_env.sh 875 } 876 877 pre_generate_files() { 878 # since make doesn't have proper dependencies, remove any possibly outdate 879 # file that might be around before generating fresh ones 880 make neat 881 if [ $QUIET -eq 1 ]; then 882 make generated_files >/dev/null 883 else 884 make generated_files 885 fi 886 } 887 888 pre_load_helpers () { 889 # Use a path relative to the currently-sourced file. 890 test_script_dir="${BASH_SOURCE%/*}" 891 source "$test_script_dir"/all-helpers.sh 892 } 893 894 ################################################################ 895 #### Termination 896 ################################################################ 897 898 post_report () { 899 msg "Done, cleaning up" 900 final_cleanup 901 902 final_report 903 } 904 905 ################################################################ 906 #### Run all the things 907 ################################################################ 908 909 # Function invoked by --error-test to test error reporting. 910 pseudo_component_error_test () { 911 msg "Testing error reporting $error_test_i" 912 if [ $KEEP_GOING -ne 0 ]; then 913 echo "Expect three failing commands." 914 fi 915 # If the component doesn't run in a subshell, changing error_test_i to an 916 # invalid integer will cause an error in the loop that runs this function. 917 error_test_i=this_should_not_be_used_since_the_component_runs_in_a_subshell 918 # Expected error: 'grep non_existent /dev/null -> 1' 919 grep non_existent /dev/null 920 # Expected error: '! grep -q . tests/scripts/all.sh -> 1' 921 not grep -q . "$0" 922 # Expected error: 'make unknown_target -> 2' 923 make unknown_target 924 false "this should not be executed" 925 } 926 927 # Run one component and clean up afterwards. 928 run_component () { 929 current_component="$1" 930 export MBEDTLS_TEST_CONFIGURATION="$current_component" 931 932 # Unconditionally create a seedfile that's sufficiently long. 933 # Do this before each component, because a previous component may 934 # have messed it up or shortened it. 935 local dd_cmd 936 dd_cmd=(dd if=/dev/urandom of=./tests/seedfile bs=64 count=1) 937 case $OSTYPE in 938 linux*|freebsd*|openbsd*) dd_cmd+=(status=none) 939 esac 940 "${dd_cmd[@]}" 941 942 if in_mbedtls_repo && in_4_x_branch; then 943 dd_cmd=(dd if=/dev/urandom of=./tf-psa-crypto/tests/seedfile bs=64 count=1) 944 case $OSTYPE in 945 linux*|freebsd*|openbsd*) dd_cmd+=(status=none) 946 esac 947 "${dd_cmd[@]}" 948 fi 949 950 if in_tf_psa_crypto_repo; then 951 pre_create_tf_psa_crypto_out_of_source_directory 952 fi 953 954 # Run the component in a subshell, with error trapping and output 955 # redirection set up based on the relevant options. 956 if [ $KEEP_GOING -eq 1 ]; then 957 # We want to keep running if the subshell fails, so 'set -e' must 958 # be off when the subshell runs. 959 set +e 960 fi 961 ( 962 if [ $QUIET -eq 1 ]; then 963 # msg() will be silenced, so just print the component name here. 964 echo "${current_component#component_}" 965 exec >/dev/null 966 fi 967 if [ $KEEP_GOING -eq 1 ]; then 968 # Keep "set -e" off, and run an ERR trap instead to record failures. 969 set -E 970 trap err_trap ERR 971 fi 972 # The next line is what runs the component 973 "$@" 974 if [ $KEEP_GOING -eq 1 ]; then 975 trap - ERR 976 exit $last_failure_status 977 fi 978 ) 979 component_status=$? 980 if [ $KEEP_GOING -eq 1 ]; then 981 set -e 982 if [ $component_status -ne 0 ]; then 983 failure_count=$((failure_count + 1)) 984 fi 985 fi 986 987 # Restore the build tree to a clean state. 988 cleanup 989 unset current_component 990 } 991 992 pre_create_tf_psa_crypto_out_of_source_directory () { 993 rm -rf "$OUT_OF_SOURCE_DIR" 994 mkdir "$OUT_OF_SOURCE_DIR" 995 } 996 997 ################################################################ 998 #### Main 999 ################################################################ 1000 1001 main () { 1002 # Preliminary setup 1003 pre_set_shell_options 1004 pre_set_signal_handlers 1005 pre_check_environment 1006 pre_load_helpers 1007 pre_load_components 1008 pre_initialize_variables 1009 pre_parse_command_line "$@" 1010 1011 setup_quiet_wrappers 1012 pre_check_git 1013 pre_restore_files 1014 pre_back_up 1015 1016 build_status=0 1017 if [ $KEEP_GOING -eq 1 ]; then 1018 pre_setup_keep_going 1019 fi 1020 pre_prepare_outcome_file 1021 pre_print_configuration 1022 pre_check_tools 1023 cleanup 1024 if in_mbedtls_repo; then 1025 pre_generate_files 1026 fi 1027 1028 # Run the requested tests. 1029 for ((error_test_i=1; error_test_i <= error_test; error_test_i++)); do 1030 run_component pseudo_component_error_test 1031 done 1032 unset error_test_i 1033 for component in $RUN_COMPONENTS; do 1034 run_component "component_$component" 1035 done 1036 1037 # We're done. 1038 post_report 1039 }