Use lcov/llvm for coverage

This commit is contained in:
Joseph Birr-Pixton 2017-06-04 17:18:48 +01:00
parent 3d874b17aa
commit e7a54f60f2
16 changed files with 122 additions and 63 deletions

3
.gitignore vendored
View File

@ -1,2 +1,5 @@
Cargo.lock
target/
*.gcda
*.gcno
*.info

View File

@ -8,8 +8,10 @@ matrix:
before_install:
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi
- if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./admin/build-kcov ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install --upgrade pip ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then admin/build-lcov ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then gem install coveralls-lcov ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libclang-common-3.8-dev llvm-3.8-dev ; fi
script:
- cargo build
- RUST_BACKTRACE=1 cargo test
@ -20,4 +22,5 @@ script:
- ( cd bogo && ./runme )
- cargo build --no-default-features
- cargo test --no-default-features --no-run
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./admin/coverage ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./admin/coverage ; fi
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then coveralls-lcov final.info ; fi

View File

@ -1,20 +0,0 @@
#!/bin/sh
#
# See https://users.rust-lang.org/t/tutorial-how-to-collect-test-coverages-for-rust-project/650
set -e
KCOV_KNOWN_GOOD=eedf728147b40a5887596d7b92f5ad94a17b3300
sudo apt-get install -y libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc binutils-dev libiberty-dev
rm -rf kcov && mkdir -p kcov
cd kcov
wget https://github.com/SimonKagstrom/kcov/archive/$KCOV_KNOWN_GOOD.tar.gz
tar xzf $KCOV_KNOWN_GOOD.tar.gz
cd kcov-$KCOV_KNOWN_GOOD
mkdir build
cd build
cmake ..
make
sudo make install

7
admin/build-lcov Executable file
View File

@ -0,0 +1,7 @@
#!/bin/sh
set -xe
if [ ! -e lcov/ ] ; then
git clone https://github.com/linux-test-project/lcov.git
cd lcov && sudo make install
fi

View File

@ -1 +0,0 @@
wrapper

View File

@ -1 +0,0 @@
wrapper

View File

@ -1 +0,0 @@
wrapper

View File

@ -1 +0,0 @@
wrapper

View File

@ -1 +0,0 @@
wrapper

View File

@ -1,6 +0,0 @@
#!/bin/bash
ME=$([[ $0 = /* ]] && echo "$0" || echo "$PWD/${0#./}")
WHICH=$(basename "$0")
HERE=$(dirname "$ME")
exec $COVERAGE $HERE/../../target/${RELEASE:-debug}/examples/$WHICH "$@"

View File

@ -1,21 +1,77 @@
#!/bin/sh
#!/usr/bin/env python
export COVERAGE_OUT=$PWD/target/coverage
export KCOV_OPTIONS="kcov --exclude-pattern=/.cargo/,intrin.h --verify"
export COVERAGE="$KCOV_OPTIONS $COVERAGE_OUT"
export RUST_TEST_THREADS=1
import os
import glob
import subprocess
# unit and integration tests. run these twice, first to capture coverage of
# child processes and then to capture coverage of the test themselves
for test in $(cargo test notexist 2>&1 | grep Running | awk '{ print $2 }') ; do
WITH_COVBIN_WRAPPERS=1 ./$test
$COVERAGE ./$test
done
LLVM_PATH = glob.glob('/usr/lib/llvm-3.8/lib/clang/3.8.[0-9]/lib/linux/')[0]
COVERAGE_OPTIONS = '-Ccodegen-units=1 -Clink-dead-code -Cpasses=insert-gcov-profiling -Zno-landing-pads -L%s -lclang_rt.profile-x86_64' % LLVM_PATH
LCOVOPTS = '--gcov-tool ./admin/llvm-gcov --rc lcov_branch_coverage=1 --rc lcov_excl_line=assert'.split()
# bogo tests
( cd bogo && ./runme )
def lcov_exe(*args):
return ['lcov'] + LCOVOPTS + list(args)
# trytls tests
( cd trytls && ./runme )
def sh(cmd):
subprocess.check_call(cmd, shell = True)
$KCOV_OPTIONS --coveralls-id=$TRAVIS_JOB_ID --report-only target/coverage/ ./target/debug/examples/tlsclient
def cleanup():
sh('cargo clean')
sh("rm -rf *.gcda *.gcno cov/ && mkdir cov/")
def run(which):
exe = filter(lambda x: '.d' not in x, glob.glob('target/debug/' + which + '-*'))[0]
sh('./' + exe)
def rustc(*args):
exe = ['cargo', 'rustc', '--all-features'] + list(args)
env = dict(os.environ)
env.update(RUSTC_WRAPPER = './admin/coverage-rustc',
COVERAGE_OPTIONS = COVERAGE_OPTIONS)
subprocess.check_call(exe, env = env)
def lcov(outfile):
exe = lcov_exe('--capture', '--directory', '.', '--base-directory', '.', '-o', outfile)
subprocess.check_call(exe)
return outfile
def merge(outfile, infiles):
arg = []
for i in infiles:
arg.append('--add')
arg.append(i)
arg.append('-o')
arg.append(outfile)
subprocess.check_call(lcov_exe(*arg))
def extract(outfile, infile):
subprocess.check_call(lcov_exe('--extract', infile, os.getcwd() + '/*', '-o', outfile))
def genhtml(outdir, infile):
subprocess.check_call(['genhtml', '--branch-coverage', '--demangle-cpp', '--legend',
infile, '-o', outdir, '--ignore-errors', 'source'])
all_infos = []
# unit tests
cleanup()
rustc('--profile', 'test', '--lib')
run('rustls')
all_infos.append(lcov('rustls.info'))
cleanup()
for example in 'tlsclient tlsserver bench bogo_shim trytls_shim'.split():
rustc('--profile', 'dev', '--example', example)
# tests
for test in 'api badssl bugs client_suites curves errors features server_suites topsites'.split():
rustc('--profile', 'dev', '--test', test)
run(test)
# trytls
sh('cd trytls && ./runme')
sh('cd bogo && ./runme')
all_infos.append(lcov('tests.info'))
merge('coverage.info', all_infos)
extract('final.info', 'coverage.info')
genhtml('target/coverage/', 'final.info')

25
admin/coverage-rustc Executable file
View File

@ -0,0 +1,25 @@
#!/bin/bash -e
get_crate_name()
{
while [[ $# -gt 1 ]] ; do
v=$1
case $v in
--crate-name)
echo $2
return
;;
esac
shift
done
}
case $(get_crate_name "$@") in
rustls|tlsclient|tlsserver|features|server_suites|client_suites|errors|api|badssl|bugs|curves|topsites|bogo_shim|trytls_shim|bench)
EXTRA=$COVERAGE_OPTIONS
;;
*)
;;
esac
exec "$@" $EXTRA

3
admin/llvm-gcov Executable file
View File

@ -0,0 +1,3 @@
#!/bin/sh -e
echo "llvm-cov gcov $*"
llvm-cov gcov $*

View File

@ -5,13 +5,15 @@
set -xe
cargo test --no-run --features dangerous_configuration
if [ ! -e ../target/debug/examples/bogo_shim ] ; then
cargo test --no-run --features dangerous_configuration
fi
if [ ! -e bogo/ ] ; then
./fetch-and-build
fi
( cd bogo && ./bogo.test -shim-path ../../admin/covbin/bogo_shim \
( cd bogo && ./bogo.test -shim-path ../../target/debug/examples/bogo_shim \
-shim-config ../config.json \
-test.parallel 1 -num-workers 1 \
-pipe -allow-unimplemented )

View File

@ -3,7 +3,6 @@ use std::process;
use std::thread;
use std::time;
use std::net;
use std::env;
extern crate regex;
use self::regex::Regex;
@ -50,19 +49,11 @@ pub fn skipped(why: &str) {
}
pub fn tlsserver_find() -> &'static str {
if env::var("WITH_COVBIN_WRAPPERS").is_ok() {
"admin/covbin/tlsserver"
} else {
"target/debug/examples/tlsserver"
}
"target/debug/examples/tlsserver"
}
pub fn tlsclient_find() -> &'static str {
if env::var("WITH_COVBIN_WRAPPERS").is_ok() {
"admin/covbin/tlsclient"
} else {
"target/debug/examples/tlsclient"
}
"target/debug/examples/tlsclient"
}
pub fn openssl_find() -> &'static str {

View File

@ -11,4 +11,4 @@ if [ ! -e trytls/ ] ; then
fi
export PYTHONPATH=trytls/lib/python2.7/site-packages/
./trytls/bin/trytls https ../admin/covbin/trytls_shim
./trytls/bin/trytls https ../target/${RELEASE:-debug}/examples/trytls_shim