From 8b6f1c4353836bae6aa683cbc65251165bd031ba Mon Sep 17 00:00:00 2001 From: Hennadii Stepanov <32963518+hebasto@users.noreply.github.com> Date: Mon, 5 Aug 2024 13:16:58 +0100 Subject: [PATCH] cmake: Add `Coverage` and `CoverageFuzz` scripts --- CMakeLists.txt | 5 ++ cmake/cov_tool_wrapper.sh.in | 5 ++ cmake/module/ProcessConfigurations.cmake | 11 ++++ cmake/script/Coverage.cmake | 77 ++++++++++++++++++++++++ cmake/script/CoverageFuzz.cmake | 42 +++++++++++++ cmake/script/CoverageInclude.cmake.in | 56 +++++++++++++++++ doc/developer-notes.md | 23 ++++--- 7 files changed, 212 insertions(+), 7 deletions(-) create mode 100644 cmake/cov_tool_wrapper.sh.in create mode 100644 cmake/script/Coverage.cmake create mode 100644 cmake/script/CoverageFuzz.cmake create mode 100644 cmake/script/CoverageInclude.cmake.in diff --git a/CMakeLists.txt b/CMakeLists.txt index 4a736da75d9..32db4189d49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,6 +463,11 @@ else() ) endif() +configure_file(cmake/script/Coverage.cmake Coverage.cmake COPYONLY) +configure_file(cmake/script/CoverageFuzz.cmake CoverageFuzz.cmake COPYONLY) +configure_file(cmake/script/CoverageInclude.cmake.in CoverageInclude.cmake @ONLY) +configure_file(contrib/filter-lcov.py filter-lcov.py COPYONLY) + # Don't allow extended (non-ASCII) symbols in identifiers. This is easier for code review. try_append_cxx_flags("-fno-extended-identifiers" TARGET core_interface SKIP_LINK) diff --git a/cmake/cov_tool_wrapper.sh.in b/cmake/cov_tool_wrapper.sh.in new file mode 100644 index 00000000000..f6b7ff3419f --- /dev/null +++ b/cmake/cov_tool_wrapper.sh.in @@ -0,0 +1,5 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +exec @COV_TOOL@ "$@" diff --git a/cmake/module/ProcessConfigurations.cmake b/cmake/module/ProcessConfigurations.cmake index 80e842f59c7..5286d10267b 100644 --- a/cmake/module/ProcessConfigurations.cmake +++ b/cmake/module/ProcessConfigurations.cmake @@ -162,3 +162,14 @@ else() endif() unset(cxx_flags_debug_overridden) endif() + +set(CMAKE_CXX_FLAGS_COVERAGE "-Og --coverage") +set(CMAKE_OBJCXX_FLAGS_COVERAGE "-Og --coverage") +set(CMAKE_EXE_LINKER_FLAGS_COVERAGE "--coverage") +set(CMAKE_SHARED_LINKER_FLAGS_COVERAGE "--coverage") +get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG) +if(is_multi_config) + if(NOT "Coverage" IN_LIST CMAKE_CONFIGURATION_TYPES) + list(APPEND CMAKE_CONFIGURATION_TYPES Coverage) + endif() +endif() diff --git a/cmake/script/Coverage.cmake b/cmake/script/Coverage.cmake new file mode 100644 index 00000000000..0df2e0b734d --- /dev/null +++ b/cmake/script/Coverage.cmake @@ -0,0 +1,77 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +include(${CMAKE_CURRENT_LIST_DIR}/CoverageInclude.cmake) + +set(functional_test_runner test/functional/test_runner.py) +if(EXTENDED_FUNCTIONAL_TESTS) + list(APPEND functional_test_runner --extended) +endif() +if(DEFINED JOBS) + list(APPEND CMAKE_CTEST_COMMAND -j ${JOBS}) + list(APPEND functional_test_runner -j ${JOBS}) +endif() + +execute_process( + COMMAND ${CMAKE_CTEST_COMMAND} --build-config Coverage + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND ${LCOV_COMMAND} --capture --directory src --test-name test_bitcoin --output-file test_bitcoin.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --zerocounters --directory src + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_FILTER_COMMAND} test_bitcoin.info test_bitcoin_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile test_bitcoin_filtered.info --output-file test_bitcoin_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile baseline_filtered.info --add-tracefile test_bitcoin_filtered.info --output-file test_bitcoin_coverage.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${GENHTML_COMMAND} test_bitcoin_coverage.info --output-directory test_bitcoin.coverage + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +execute_process( + COMMAND ${functional_test_runner} + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND ${LCOV_COMMAND} --capture --directory src --test-name functional-tests --output-file functional_test.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --zerocounters --directory src + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_FILTER_COMMAND} functional_test.info functional_test_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile functional_test_filtered.info --output-file functional_test_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile baseline_filtered.info --add-tracefile test_bitcoin_filtered.info --add-tracefile functional_test_filtered.info --output-file total_coverage.info + COMMAND ${GREP_EXECUTABLE} "%" + COMMAND ${AWK_EXECUTABLE} "{ print substr($3,2,50) \"/\" $5 }" + OUTPUT_FILE coverage_percent.txt + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${GENHTML_COMMAND} total_coverage.info --output-directory total.coverage + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) diff --git a/cmake/script/CoverageFuzz.cmake b/cmake/script/CoverageFuzz.cmake new file mode 100644 index 00000000000..2626ea0cb5d --- /dev/null +++ b/cmake/script/CoverageFuzz.cmake @@ -0,0 +1,42 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +include(${CMAKE_CURRENT_LIST_DIR}/CoverageInclude.cmake) + +if(NOT DEFINED FUZZ_SEED_CORPUS_DIR) + set(FUZZ_SEED_CORPUS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/qa-assets/fuzz_seed_corpus) +endif() + +execute_process( + COMMAND test/fuzz/test_runner.py ${FUZZ_SEED_CORPUS_DIR} --loglevel DEBUG + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND ${LCOV_COMMAND} --capture --directory src --test-name fuzz-tests --output-file fuzz.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --zerocounters --directory src + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_FILTER_COMMAND} fuzz.info fuzz_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile fuzz_filtered.info --output-file fuzz_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile baseline_filtered.info --add-tracefile fuzz_filtered.info --output-file fuzz_coverage.info + COMMAND ${GREP_EXECUTABLE} "%" + COMMAND ${AWK_EXECUTABLE} "{ print substr($3,2,50) \"/\" $5 }" + OUTPUT_FILE coverage_percent.txt + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${GENHTML_COMMAND} fuzz_coverage.info --output-directory fuzz.coverage + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) diff --git a/cmake/script/CoverageInclude.cmake.in b/cmake/script/CoverageInclude.cmake.in new file mode 100644 index 00000000000..7a8bf2f0af2 --- /dev/null +++ b/cmake/script/CoverageInclude.cmake.in @@ -0,0 +1,56 @@ +# Copyright (c) 2024-present The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or https://opensource.org/license/mit/. + +if("@CMAKE_CXX_COMPILER_ID@" STREQUAL "Clang") + find_program(LLVM_COV_EXECUTABLE llvm-cov REQUIRED) + set(COV_TOOL "${LLVM_COV_EXECUTABLE} gcov") +else() + find_program(GCOV_EXECUTABLE gcov REQUIRED) + set(COV_TOOL "${GCOV_EXECUTABLE}") +endif() + +# COV_TOOL is used to replace a placeholder. +configure_file( + cmake/cov_tool_wrapper.sh.in ${CMAKE_CURRENT_LIST_DIR}/cov_tool_wrapper.sh + FILE_PERMISSIONS OWNER_READ OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE + WORLD_READ + @ONLY +) + +find_program(LCOV_EXECUTABLE lcov REQUIRED) +separate_arguments(LCOV_OPTS) +set(LCOV_COMMAND ${LCOV_EXECUTABLE} --gcov-tool ${CMAKE_CURRENT_LIST_DIR}/cov_tool_wrapper.sh ${LCOV_OPTS}) + +find_program(GENHTML_EXECUTABLE genhtml REQUIRED) +set(GENHTML_COMMAND ${GENHTML_EXECUTABLE} --show-details ${LCOV_OPTS}) + +find_program(GREP_EXECUTABLE grep REQUIRED) +find_program(AWK_EXECUTABLE awk REQUIRED) + +set(LCOV_FILTER_COMMAND ./filter-lcov.py) +list(APPEND LCOV_FILTER_COMMAND -p "/usr/local/") +list(APPEND LCOV_FILTER_COMMAND -p "/usr/include/") +list(APPEND LCOV_FILTER_COMMAND -p "/usr/lib/") +list(APPEND LCOV_FILTER_COMMAND -p "/usr/lib64/") +list(APPEND LCOV_FILTER_COMMAND -p "src/leveldb/") +list(APPEND LCOV_FILTER_COMMAND -p "src/crc32c/") +list(APPEND LCOV_FILTER_COMMAND -p "src/bench/") +list(APPEND LCOV_FILTER_COMMAND -p "src/crypto/ctaes") +list(APPEND LCOV_FILTER_COMMAND -p "src/minisketch") +list(APPEND LCOV_FILTER_COMMAND -p "src/secp256k1") +list(APPEND LCOV_FILTER_COMMAND -p "depends") + +execute_process( + COMMAND ${LCOV_COMMAND} --capture --initial --directory src --output-file baseline.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_FILTER_COMMAND} baseline.info baseline_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) +execute_process( + COMMAND ${LCOV_COMMAND} --add-tracefile baseline_filtered.info --output-file baseline_filtered.info + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) diff --git a/doc/developer-notes.md b/doc/developer-notes.md index 93de6ce671b..3066aabe600 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -476,25 +476,34 @@ $ ./test/functional/test_runner.py --valgrind ### Compiling for test coverage -LCOV can be used to generate a test coverage report based upon `make check` +LCOV can be used to generate a test coverage report based upon `ctest` execution. LCOV must be installed on your system (e.g. the `lcov` package on Debian/Ubuntu). To enable LCOV report generation during test runs: ```shell -./configure --enable-lcov -make -make cov +cmake -B build -DCMAKE_BUILD_TYPE=Coverage +cmake --build build +cmake -P build/Coverage.cmake -# A coverage report will now be accessible at `./test_bitcoin.coverage/index.html`, -# which covers unit tests, and `./total.coverage/index.html`, which covers +# A coverage report will now be accessible at `./build/test_bitcoin.coverage/index.html`, +# which covers unit tests, and `./build/total.coverage/index.html`, which covers # unit and functional tests. ``` Additional LCOV options can be specified using `LCOV_OPTS`, but may be dependent on the version of LCOV. For example, when using LCOV `2.x`, branch coverage can be -enabled by setting `LCOV_OPTS="--rc branch_coverage=1"`, when configuring. +enabled by setting `LCOV_OPTS="--rc branch_coverage=1"`: + +``` +cmake -DLCOV_OPTS="--rc branch_coverage=1" -P build/Coverage.cmake +``` + +To enable test parallelism: +``` +cmake -DJOBS=$(nproc) -P build/Coverage.cmake +``` ### Performance profiling with perf