diff --git a/.gitignore b/.gitignore index 1e7caa9e..37d537ea 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ Cargo.lock target/ +*.gcda +*.gcno +*.info diff --git a/.travis.yml b/.travis.yml index 1a2114f5..d3a5633a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,10 @@ matrix: before_install: - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update ; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew install openssl ; fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./admin/build-kcov ; fi - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install --upgrade pip ; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then admin/build-lcov ; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then gem install coveralls-lcov ; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libclang-common-3.8-dev llvm-3.8-dev ; fi script: - cargo build - RUST_BACKTRACE=1 cargo test @@ -20,4 +22,5 @@ script: - ( cd bogo && ./runme ) - cargo build --no-default-features - cargo test --no-default-features --no-run - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./admin/coverage ; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./admin/coverage ; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then coveralls-lcov final.info ; fi diff --git a/admin/build-kcov b/admin/build-kcov deleted file mode 100755 index ab814f83..00000000 --- a/admin/build-kcov +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh -# -# See https://users.rust-lang.org/t/tutorial-how-to-collect-test-coverages-for-rust-project/650 - -set -e - -KCOV_KNOWN_GOOD=eedf728147b40a5887596d7b92f5ad94a17b3300 - -sudo apt-get install -y libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc binutils-dev libiberty-dev - -rm -rf kcov && mkdir -p kcov -cd kcov -wget https://github.com/SimonKagstrom/kcov/archive/$KCOV_KNOWN_GOOD.tar.gz -tar xzf $KCOV_KNOWN_GOOD.tar.gz -cd kcov-$KCOV_KNOWN_GOOD -mkdir build -cd build -cmake .. -make -sudo make install diff --git a/admin/build-lcov b/admin/build-lcov new file mode 100755 index 00000000..70ee3c52 --- /dev/null +++ b/admin/build-lcov @@ -0,0 +1,7 @@ +#!/bin/sh +set -xe + +if [ ! -e lcov/ ] ; then + git clone https://github.com/linux-test-project/lcov.git + cd lcov && sudo make install +fi diff --git a/admin/covbin/bench b/admin/covbin/bench deleted file mode 120000 index 22cb46f8..00000000 --- a/admin/covbin/bench +++ /dev/null @@ -1 +0,0 @@ -wrapper \ No newline at end of file diff --git a/admin/covbin/bogo_shim b/admin/covbin/bogo_shim deleted file mode 120000 index 22cb46f8..00000000 --- a/admin/covbin/bogo_shim +++ /dev/null @@ -1 +0,0 @@ -wrapper \ No newline at end of file diff --git a/admin/covbin/tlsclient b/admin/covbin/tlsclient deleted file mode 120000 index 22cb46f8..00000000 --- a/admin/covbin/tlsclient +++ /dev/null @@ -1 +0,0 @@ -wrapper \ No newline at end of file diff --git a/admin/covbin/tlsserver b/admin/covbin/tlsserver deleted file mode 120000 index 22cb46f8..00000000 --- a/admin/covbin/tlsserver +++ /dev/null @@ -1 +0,0 @@ -wrapper \ No newline at end of file diff --git a/admin/covbin/trytls_shim b/admin/covbin/trytls_shim deleted file mode 120000 index 22cb46f8..00000000 --- a/admin/covbin/trytls_shim +++ /dev/null @@ -1 +0,0 @@ -wrapper \ No newline at end of file diff --git a/admin/covbin/wrapper b/admin/covbin/wrapper deleted file mode 100755 index 274139af..00000000 --- a/admin/covbin/wrapper +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -ME=$([[ $0 = /* ]] && echo "$0" || echo "$PWD/${0#./}") -WHICH=$(basename "$0") -HERE=$(dirname "$ME") -exec $COVERAGE $HERE/../../target/${RELEASE:-debug}/examples/$WHICH "$@" diff --git a/admin/coverage b/admin/coverage index cd28d418..66eb17e1 100755 --- a/admin/coverage +++ b/admin/coverage @@ -1,21 +1,77 @@ -#!/bin/sh +#!/usr/bin/env python -export COVERAGE_OUT=$PWD/target/coverage -export KCOV_OPTIONS="kcov --exclude-pattern=/.cargo/,intrin.h --verify" -export COVERAGE="$KCOV_OPTIONS $COVERAGE_OUT" -export RUST_TEST_THREADS=1 +import os +import glob +import subprocess -# unit and integration tests. run these twice, first to capture coverage of -# child processes and then to capture coverage of the test themselves -for test in $(cargo test notexist 2>&1 | grep Running | awk '{ print $2 }') ; do - WITH_COVBIN_WRAPPERS=1 ./$test - $COVERAGE ./$test -done +LLVM_PATH = glob.glob('/usr/lib/llvm-3.8/lib/clang/3.8.[0-9]/lib/linux/')[0] +COVERAGE_OPTIONS = '-Ccodegen-units=1 -Clink-dead-code -Cpasses=insert-gcov-profiling -Zno-landing-pads -L%s -lclang_rt.profile-x86_64' % LLVM_PATH +LCOVOPTS = '--gcov-tool ./admin/llvm-gcov --rc lcov_branch_coverage=1 --rc lcov_excl_line=assert'.split() -# bogo tests -( cd bogo && ./runme ) +def lcov_exe(*args): + return ['lcov'] + LCOVOPTS + list(args) -# trytls tests -( cd trytls && ./runme ) +def sh(cmd): + subprocess.check_call(cmd, shell = True) -$KCOV_OPTIONS --coveralls-id=$TRAVIS_JOB_ID --report-only target/coverage/ ./target/debug/examples/tlsclient +def cleanup(): + sh('cargo clean') + sh("rm -rf *.gcda *.gcno cov/ && mkdir cov/") + +def run(which): + exe = filter(lambda x: '.d' not in x, glob.glob('target/debug/' + which + '-*'))[0] + sh('./' + exe) + +def rustc(*args): + exe = ['cargo', 'rustc', '--all-features'] + list(args) + env = dict(os.environ) + env.update(RUSTC_WRAPPER = './admin/coverage-rustc', + COVERAGE_OPTIONS = COVERAGE_OPTIONS) + subprocess.check_call(exe, env = env) + +def lcov(outfile): + exe = lcov_exe('--capture', '--directory', '.', '--base-directory', '.', '-o', outfile) + subprocess.check_call(exe) + return outfile + +def merge(outfile, infiles): + arg = [] + for i in infiles: + arg.append('--add') + arg.append(i) + arg.append('-o') + arg.append(outfile) + subprocess.check_call(lcov_exe(*arg)) + +def extract(outfile, infile): + subprocess.check_call(lcov_exe('--extract', infile, os.getcwd() + '/*', '-o', outfile)) + +def genhtml(outdir, infile): + subprocess.check_call(['genhtml', '--branch-coverage', '--demangle-cpp', '--legend', + infile, '-o', outdir, '--ignore-errors', 'source']) + +all_infos = [] + +# unit tests +cleanup() +rustc('--profile', 'test', '--lib') +run('rustls') +all_infos.append(lcov('rustls.info')) + +cleanup() +for example in 'tlsclient tlsserver bench bogo_shim trytls_shim'.split(): + rustc('--profile', 'dev', '--example', example) + +# tests +for test in 'api badssl bugs client_suites curves errors features server_suites topsites'.split(): + rustc('--profile', 'dev', '--test', test) + run(test) + +# trytls +sh('cd trytls && ./runme') +sh('cd bogo && ./runme') +all_infos.append(lcov('tests.info')) + +merge('coverage.info', all_infos) +extract('final.info', 'coverage.info') +genhtml('target/coverage/', 'final.info') diff --git a/admin/coverage-rustc b/admin/coverage-rustc new file mode 100755 index 00000000..a1360503 --- /dev/null +++ b/admin/coverage-rustc @@ -0,0 +1,25 @@ +#!/bin/bash -e + +get_crate_name() +{ + while [[ $# -gt 1 ]] ; do + v=$1 + case $v in + --crate-name) + echo $2 + return + ;; + esac + shift + done +} + +case $(get_crate_name "$@") in + rustls|tlsclient|tlsserver|features|server_suites|client_suites|errors|api|badssl|bugs|curves|topsites|bogo_shim|trytls_shim|bench) + EXTRA=$COVERAGE_OPTIONS + ;; + *) + ;; +esac + +exec "$@" $EXTRA diff --git a/admin/llvm-gcov b/admin/llvm-gcov new file mode 100755 index 00000000..4e7a811a --- /dev/null +++ b/admin/llvm-gcov @@ -0,0 +1,3 @@ +#!/bin/sh -e +echo "llvm-cov gcov $*" +llvm-cov gcov $* diff --git a/bogo/runme b/bogo/runme index 5104f46d..e09a3ce8 100755 --- a/bogo/runme +++ b/bogo/runme @@ -5,13 +5,15 @@ set -xe -cargo test --no-run --features dangerous_configuration +if [ ! -e ../target/debug/examples/bogo_shim ] ; then + cargo test --no-run --features dangerous_configuration +fi if [ ! -e bogo/ ] ; then ./fetch-and-build fi -( cd bogo && ./bogo.test -shim-path ../../admin/covbin/bogo_shim \ +( cd bogo && ./bogo.test -shim-path ../../target/debug/examples/bogo_shim \ -shim-config ../config.json \ -test.parallel 1 -num-workers 1 \ -pipe -allow-unimplemented ) diff --git a/tests/common/mod.rs b/tests/common/mod.rs index ee822121..331e9c60 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -3,7 +3,6 @@ use std::process; use std::thread; use std::time; use std::net; -use std::env; extern crate regex; use self::regex::Regex; @@ -50,19 +49,11 @@ pub fn skipped(why: &str) { } pub fn tlsserver_find() -> &'static str { - if env::var("WITH_COVBIN_WRAPPERS").is_ok() { - "admin/covbin/tlsserver" - } else { - "target/debug/examples/tlsserver" - } + "target/debug/examples/tlsserver" } pub fn tlsclient_find() -> &'static str { - if env::var("WITH_COVBIN_WRAPPERS").is_ok() { - "admin/covbin/tlsclient" - } else { - "target/debug/examples/tlsclient" - } + "target/debug/examples/tlsclient" } pub fn openssl_find() -> &'static str { diff --git a/trytls/runme b/trytls/runme index f8356da1..85521cd8 100755 --- a/trytls/runme +++ b/trytls/runme @@ -11,4 +11,4 @@ if [ ! -e trytls/ ] ; then fi export PYTHONPATH=trytls/lib/python2.7/site-packages/ -./trytls/bin/trytls https ../admin/covbin/trytls_shim +./trytls/bin/trytls https ../target/${RELEASE:-debug}/examples/trytls_shim