outcome-analysis.sh (3730B)
1 #!/bin/sh 2 3 # This script runs tests before and after a PR and analyzes the results in 4 # order to highlight any difference in the set of tests skipped. 5 # 6 # It can be used to check the first testing criterion mentioned in strategy.md, 7 # end of section "Supporting builds with drivers without the software 8 # implementation", namely: the sets of tests skipped in the default config and 9 # the full config must be the same before and after the PR. 10 # 11 # USAGE: 12 # - First, commit any uncommited changes. (Also, see warning below.) 13 # - Then launch --> [SKIP_SSL_OPT=1] docs/architecture/psa-migration/outcome-analysis.sh 14 # - SKIP_SSL_OPT=1 can optionally be set to skip ssl-opt.sh tests 15 # 16 # WARNING: this script checks out a commit other than the head of the current 17 # branch; it checks out the current branch again when running successfully, 18 # but while the script is running, or if it terminates early in error, you 19 # should be aware that you might be at a different commit than expected. 20 # 21 # NOTE: you can comment out parts that don't need to be re-done when 22 # re-running this script (for example "get numbers before this PR"). 23 24 set -eu 25 26 : ${SKIP_SSL_OPT:=0} 27 28 cleanup() { 29 make clean 30 git checkout -- include/mbedtls/mbedtls_config.h include/psa/crypto_config.h 31 } 32 33 record() { 34 export MBEDTLS_TEST_OUTCOME_FILE="$PWD/outcome-$1.csv" 35 rm -f $MBEDTLS_TEST_OUTCOME_FILE 36 37 make check 38 39 if [ $SKIP_SSL_OPT -eq 0 ]; then 40 make -C programs ssl/ssl_server2 ssl/ssl_client2 \ 41 test/udp_proxy test/query_compile_time_config 42 tests/ssl-opt.sh 43 fi 44 } 45 46 # save current HEAD. 47 # Note: this can optionally be updated to 48 # HEAD=$(git branch --show-current) 49 # when using a Git version above 2.22 50 HEAD=$(git rev-parse --abbrev-ref HEAD) 51 52 # get the numbers before this PR for default and full 53 cleanup 54 git checkout $(git merge-base HEAD development) 55 56 record "before-default" 57 58 cleanup 59 60 scripts/config.py full 61 record "before-full" 62 63 # get the numbers now for default and full 64 cleanup 65 git checkout $HEAD 66 67 record "after-default" 68 69 cleanup 70 71 scripts/config.py full 72 record "after-full" 73 74 cleanup 75 76 # analysis 77 78 populate_suites () { 79 SUITES='' 80 make generated_files >/dev/null 81 data_files=$(cd tests/suites && echo *.data) 82 for data in $data_files; do 83 suite=${data%.data} 84 SUITES="$SUITES $suite" 85 done 86 make neat 87 88 if [ $SKIP_SSL_OPT -eq 0 ]; then 89 SUITES="$SUITES ssl-opt" 90 extra_files=$(cd tests/opt-testcases && echo *.sh) 91 for extra in $extra_files; do 92 suite=${extra%.sh} 93 SUITES="$SUITES $suite" 94 done 95 fi 96 } 97 98 compare_suite () { 99 ref="outcome-$1.csv" 100 new="outcome-$2.csv" 101 suite="$3" 102 103 pattern_suite=";$suite;" 104 total=$(grep -c "$pattern_suite" "$ref") 105 sed_cmd="s/^.*$pattern_suite\(.*\);SKIP.*/\1/p" 106 sed -n "$sed_cmd" "$ref" > skipped-ref 107 sed -n "$sed_cmd" "$new" > skipped-new 108 nb_ref=$(wc -l <skipped-ref) 109 nb_new=$(wc -l <skipped-new) 110 111 name=${suite#test_suite_} 112 printf "%40s: total %4d; skipped %4d -> %4d\n" \ 113 $name $total $nb_ref $nb_new 114 if diff skipped-ref skipped-new | grep '^> '; then 115 ret=1 116 else 117 ret=0 118 fi 119 rm skipped-ref skipped-new 120 return $ret 121 } 122 123 compare_builds () { 124 printf "\n*** Comparing $1 -> $2 ***\n" 125 failed='' 126 for suite in $SUITES; do 127 if compare_suite "$1" "$2" "$suite"; then :; else 128 failed="$failed $suite" 129 fi 130 done 131 if [ -z "$failed" ]; then 132 printf "No coverage gap found.\n" 133 else 134 printf "Suites with less coverage:%s\n" "$failed" 135 fi 136 } 137 138 populate_suites 139 compare_builds before-default after-default 140 compare_builds before-full after-full