basic-build-test.sh (8411B)
1 #!/bin/sh 2 3 # basic-build-test.sh 4 # 5 # Copyright The Mbed TLS Contributors 6 # SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 7 # 8 # Purpose 9 # 10 # Executes the basic test suites, captures the results, and generates a simple 11 # test report and code coverage report. 12 # 13 # The tests include: 14 # * Unit tests - executed using tests/scripts/run-test-suite.pl 15 # * Self-tests - executed using the test suites above 16 # * System tests - executed using tests/ssl-opt.sh 17 # * Interoperability tests - executed using tests/compat.sh 18 # 19 # The tests focus on functionality and do not consider performance. 20 # 21 # Note the tests self-adapt due to configurations in include/mbedtls/mbedtls_config.h 22 # which can lead to some tests being skipped, and can cause the number of 23 # available tests to fluctuate. 24 # 25 # This script has been written to be generic and should work on any shell. 26 # 27 # Usage: basic-build-test.sh 28 # 29 30 # Abort on errors (and uninitiliased variables) 31 set -eu 32 33 if [ -d library -a -d include -a -d tests ]; then :; else 34 echo "Must be run from Mbed TLS root" >&2 35 exit 1 36 fi 37 38 : ${OPENSSL:="openssl"} 39 : ${GNUTLS_CLI:="gnutls-cli"} 40 : ${GNUTLS_SERV:="gnutls-serv"} 41 42 # Used to make ssl-opt.sh deterministic. 43 # 44 # See also RELEASE_SEED in all.sh. Debugging is easier if both values are kept 45 # in sync. If you change the value here because it breaks some tests, you'll 46 # definitely want to change it in all.sh as well. 47 : ${SEED:=1} 48 export SEED 49 50 # if MAKEFLAGS is not set add the -j option to speed up invocations of make 51 if [ -z "${MAKEFLAGS+set}" ]; then 52 export MAKEFLAGS="-j" 53 fi 54 55 # To avoid setting OpenSSL and GnuTLS for each call to compat.sh and ssl-opt.sh 56 # we just export the variables they require 57 export OPENSSL="$OPENSSL" 58 export GNUTLS_CLI="$GNUTLS_CLI" 59 export GNUTLS_SERV="$GNUTLS_SERV" 60 61 CONFIG_H='include/mbedtls/mbedtls_config.h' 62 CONFIG_BAK="$CONFIG_H.bak" 63 64 # Step 0 - print build environment info 65 OPENSSL="$OPENSSL" \ 66 GNUTLS_CLI="$GNUTLS_CLI" \ 67 GNUTLS_SERV="$GNUTLS_SERV" \ 68 framework/scripts/output_env.sh 69 echo 70 71 # Step 1 - Make and instrumented build for code coverage 72 export CFLAGS=' --coverage -g3 -O0 ' 73 export LDFLAGS=' --coverage' 74 make clean 75 cp "$CONFIG_H" "$CONFIG_BAK" 76 scripts/config.py full 77 make 78 79 80 # Step 2 - Execute the tests 81 TEST_OUTPUT=out_${PPID} 82 cd tests 83 if [ ! -f "seedfile" ]; then 84 dd if=/dev/urandom of="seedfile" bs=64 count=1 85 fi 86 echo 87 88 # Step 2a - Unit Tests (keep going even if some tests fail) 89 echo '################ Unit tests ################' 90 perl scripts/run-test-suites.pl -v 2 |tee unit-test-$TEST_OUTPUT 91 echo '^^^^^^^^^^^^^^^^ Unit tests ^^^^^^^^^^^^^^^^' 92 echo 93 94 # Step 2b - System Tests (keep going even if some tests fail) 95 echo 96 echo '################ ssl-opt.sh ################' 97 echo "ssl-opt.sh will use SEED=$SEED for udp_proxy" 98 sh ssl-opt.sh |tee sys-test-$TEST_OUTPUT 99 echo '^^^^^^^^^^^^^^^^ ssl-opt.sh ^^^^^^^^^^^^^^^^' 100 echo 101 102 # Step 2c - Compatibility tests (keep going even if some tests fail) 103 echo '################ compat.sh ################' 104 { 105 echo '#### compat.sh: Default versions' 106 sh compat.sh -e 'ARIA\|CHACHA' 107 echo 108 109 echo '#### compat.sh: next (ARIA, ChaCha)' 110 OPENSSL="$OPENSSL_NEXT" sh compat.sh -e '^$' -f 'ARIA\|CHACHA' 111 echo 112 } | tee compat-test-$TEST_OUTPUT 113 echo '^^^^^^^^^^^^^^^^ compat.sh ^^^^^^^^^^^^^^^^' 114 echo 115 116 # Step 3 - Process the coverage report 117 cd .. 118 { 119 make lcov 120 echo SUCCESS 121 } | tee tests/cov-$TEST_OUTPUT 122 123 if [ "$(tail -n1 tests/cov-$TEST_OUTPUT)" != "SUCCESS" ]; then 124 echo >&2 "Fatal: 'make lcov' failed" 125 exit 2 126 fi 127 128 129 # Step 4 - Summarise the test report 130 echo 131 echo "=========================================================================" 132 echo "Test Report Summary" 133 echo 134 135 # A failure of the left-hand side of a pipe is ignored (this is a limitation 136 # of sh). We'll use the presence of this file as a marker that the generation 137 # of the report succeeded. 138 rm -f "tests/basic-build-test-$$.ok" 139 140 { 141 142 cd tests 143 144 # Step 4a - Unit tests 145 echo "Unit tests - tests/scripts/run-test-suites.pl" 146 147 PASSED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/test cases passed :[\t]*\([0-9]*\)/\1/p'| tr -d ' ') 148 SKIPPED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/skipped :[ \t]*\([0-9]*\)/\1/p'| tr -d ' ') 149 TOTAL_SUITES=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) .*, [0-9]* tests run)/\1/p'| tr -d ' ') 150 FAILED_TESTS=$(tail -n6 unit-test-$TEST_OUTPUT|sed -n -e 's/failed :[\t]*\([0-9]*\)/\1/p' |tr -d ' ') 151 152 echo "No test suites : $TOTAL_SUITES" 153 echo "Passed : $PASSED_TESTS" 154 echo "Failed : $FAILED_TESTS" 155 echo "Skipped : $SKIPPED_TESTS" 156 echo "Total exec'd tests : $(($PASSED_TESTS + $FAILED_TESTS))" 157 echo "Total avail tests : $(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS))" 158 echo 159 160 TOTAL_PASS=$PASSED_TESTS 161 TOTAL_FAIL=$FAILED_TESTS 162 TOTAL_SKIP=$SKIPPED_TESTS 163 TOTAL_AVAIL=$(($PASSED_TESTS + $FAILED_TESTS + $SKIPPED_TESTS)) 164 TOTAL_EXED=$(($PASSED_TESTS + $FAILED_TESTS)) 165 166 # Step 4b - TLS Options tests 167 echo "TLS Options tests - tests/ssl-opt.sh" 168 169 PASSED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p') 170 SKIPPED_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p') 171 TOTAL_TESTS=$(tail -n5 sys-test-$TEST_OUTPUT|sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p') 172 FAILED_TESTS=$(($TOTAL_TESTS - $PASSED_TESTS)) 173 174 echo "Passed : $PASSED_TESTS" 175 echo "Failed : $FAILED_TESTS" 176 echo "Skipped : $SKIPPED_TESTS" 177 echo "Total exec'd tests : $TOTAL_TESTS" 178 echo "Total avail tests : $(($TOTAL_TESTS + $SKIPPED_TESTS))" 179 echo 180 181 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 182 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 183 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 184 TOTAL_AVAIL=$(($TOTAL_AVAIL + $TOTAL_TESTS + $SKIPPED_TESTS)) 185 TOTAL_EXED=$(($TOTAL_EXED + $TOTAL_TESTS)) 186 187 188 # Step 4c - System Compatibility tests 189 echo "System/Compatibility tests - tests/compat.sh" 190 191 PASSED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* (\([0-9]*\) \/ [0-9]* tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 192 SKIPPED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ [0-9]* tests (\([0-9]*\) skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 193 EXED_TESTS=$(cat compat-test-$TEST_OUTPUT | sed -n -e 's/.* ([0-9]* \/ \([0-9]*\) tests ([0-9]* skipped))$/\1/p' | awk 'BEGIN{ s = 0 } { s += $1 } END{ print s }') 194 FAILED_TESTS=$(($EXED_TESTS - $PASSED_TESTS)) 195 196 echo "Passed : $PASSED_TESTS" 197 echo "Failed : $FAILED_TESTS" 198 echo "Skipped : $SKIPPED_TESTS" 199 echo "Total exec'd tests : $EXED_TESTS" 200 echo "Total avail tests : $(($EXED_TESTS + $SKIPPED_TESTS))" 201 echo 202 203 TOTAL_PASS=$(($TOTAL_PASS+$PASSED_TESTS)) 204 TOTAL_FAIL=$(($TOTAL_FAIL+$FAILED_TESTS)) 205 TOTAL_SKIP=$(($TOTAL_SKIP+$SKIPPED_TESTS)) 206 TOTAL_AVAIL=$(($TOTAL_AVAIL + $EXED_TESTS + $SKIPPED_TESTS)) 207 TOTAL_EXED=$(($TOTAL_EXED + $EXED_TESTS)) 208 209 210 # Step 4d - Grand totals 211 echo "-------------------------------------------------------------------------" 212 echo "Total tests" 213 214 echo "Total Passed : $TOTAL_PASS" 215 echo "Total Failed : $TOTAL_FAIL" 216 echo "Total Skipped : $TOTAL_SKIP" 217 echo "Total exec'd tests : $TOTAL_EXED" 218 echo "Total avail tests : $TOTAL_AVAIL" 219 echo 220 221 222 # Step 4e - Coverage report 223 echo "Coverage statistics:" 224 sed -n '1,/^Overall coverage/d; /%/p' cov-$TEST_OUTPUT 225 echo 226 227 rm unit-test-$TEST_OUTPUT 228 rm sys-test-$TEST_OUTPUT 229 rm compat-test-$TEST_OUTPUT 230 rm cov-$TEST_OUTPUT 231 232 # Mark the report generation as having succeeded. This must be the 233 # last thing in the report generation. 234 touch "basic-build-test-$$.ok" 235 } | tee coverage-summary.txt 236 237 make clean 238 239 if [ -f "$CONFIG_BAK" ]; then 240 mv "$CONFIG_BAK" "$CONFIG_H" 241 fi 242 243 # The file must exist, otherwise it means something went wrong while generating 244 # the coverage report. If something did go wrong, rm will complain so this 245 # script will exit with a failure status. 246 rm "tests/basic-build-test-$$.ok"