From 4d2a926cb40995e483846cc125e42bdbcc1f5918 Mon Sep 17 00:00:00 2001 From: dexX7 Date: Thu, 28 May 2015 01:28:39 +0200 Subject: [PATCH 1/6] Ignore coverage data related and temporary test files --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a8035731d17..2b6b19883d3 100644 --- a/.gitignore +++ b/.gitignore @@ -94,6 +94,7 @@ build #lcov *.gcno +*.gcda /*.info test_bitcoin.coverage/ total.coverage/ @@ -107,6 +108,9 @@ qa/pull-tester/run-bitcoind-for-test.sh qa/pull-tester/tests_config.py qa/pull-tester/cache/* qa/pull-tester/test.*/* +qa/tmp +cache/ +share/BitcoindComparisonTool.jar !src/leveldb*/Makefile From d425877557037b063c5cadcc49ef0582706cff77 Mon Sep 17 00:00:00 2001 From: dexX7 Date: Thu, 28 May 2015 02:47:53 +0200 Subject: [PATCH 2/6] Remove coverage and test related files, when cleaning up Until now there were quite a few leftovers, and only the coverage related files in `src/` were cleaned, while the ones in the other dirs remained. `qa/tmp/` is related to the BitcoinJ tests, and `cache/` is related to RPC tests. --- Makefile.am | 2 +- src/Makefile.am | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 8a7140398f8..93a768b1c19 100644 --- a/Makefile.am +++ b/Makefile.am @@ -211,4 +211,4 @@ CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER) .INTERMEDIATE: $(COVERAGE_INFO) clean-local: - rm -rf test_bitcoin.coverage/ total.coverage/ $(OSX_APP) + rm -rf coverage_percent.txt test_bitcoin.coverage/ total.coverage/ qa/tmp/ cache/ $(OSX_APP) diff --git a/src/Makefile.am b/src/Makefile.am index 312643cec36..f35b9dc8981 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -412,7 +412,19 @@ libbitcoinconsensus_la_CPPFLAGS = $(CRYPTO_CFLAGS) -I$(builddir)/obj -DBUILD_BIT endif # -CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a *.gcda *.gcno +CLEANFILES = leveldb/libleveldb.a leveldb/libmemenv.a +CLEANFILES += *.gcda *.gcno +CLEANFILES += compat/*.gcda compat/*.gcno +CLEANFILES += consensus/*.gcda consensus/*.gcno +CLEANFILES += crypto/*.gcda crypto/*.gcno +CLEANFILES += policy/*.gcda policy/*.gcno +CLEANFILES += primitives/*.gcda primitives/*.gcno +CLEANFILES += script/*.gcda script/*.gcno +CLEANFILES += support/*.gcda support/*.gcno +CLEANFILES += univalue/*.gcda univalue/*.gcno +CLEANFILES += wallet/*.gcda wallet/*.gcno +CLEANFILES += wallet/test/*.gcda wallet/test/*.gcno +CLEANFILES += zmq/*.gcda zmq/*.gcno DISTCLEANFILES = obj/build.h @@ -422,7 +434,7 @@ clean-local: -$(MAKE) -C leveldb clean -$(MAKE) -C secp256k1 clean -$(MAKE) -C univalue clean - rm -f leveldb/*/*.gcno leveldb/helpers/memenv/*.gcno + -rm -f leveldb/*/*.gcda leveldb/*/*.gcno leveldb/helpers/memenv/*.gcda leveldb/helpers/memenv/*.gcno -rm -f config.h .rc.o: From 8e3a27bbbfc8d453877bf6521044c64dfbf64610 Mon Sep 17 00:00:00 2001 From: dexX7 Date: Sun, 11 Oct 2015 22:05:20 +0200 Subject: [PATCH 3/6] Require Python for RPC tests, when using lcov Because Python is (going to be) used to run the RPC tests, when gathering coverage data with lcov, it is explicitly checked, whether Python is really available. --- configure.ac | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/configure.ac b/configure.ac index 4318aafa559..13598acd2f1 100644 --- a/configure.ac +++ b/configure.ac @@ -58,6 +58,7 @@ AC_PATH_TOOL(STRIP, strip) AC_PATH_TOOL(GCOV, gcov) AC_PATH_PROG(LCOV, lcov) AC_PATH_PROG(JAVA, java) +AC_PATH_PROG(PYTHON, python) AC_PATH_PROG(GENHTML, genhtml) AC_PATH_PROG([GIT], [git]) AC_PATH_PROG(CCACHE,ccache) @@ -351,6 +352,9 @@ if test x$use_lcov = xyes; then if test x$JAVA = x; then AC_MSG_ERROR("lcov testing requested but java not found") fi + if test x$PYTHON = x; then + AC_MSG_ERROR("lcov testing requested but python not found") + fi if test x$GENHTML = x; then AC_MSG_ERROR("lcov testing requested but genhtml not found") fi From 45d4ff0c2094813e8a1c5446e728449157a289b8 Mon Sep 17 00:00:00 2001 From: dexX7 Date: Mon, 12 Oct 2015 14:37:57 +0200 Subject: [PATCH 4/6] Add config option to enable extended RPC tests for code coverage When using lcov to gather code coverage data, the configuration option `--enable-extended-rpc-tests` may be used to enable extended RPC tests. --- configure.ac | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/configure.ac b/configure.ac index 13598acd2f1..25ace88f327 100644 --- a/configure.ac +++ b/configure.ac @@ -107,6 +107,11 @@ AC_ARG_ENABLE([comparison-tool-reorg-tests], [use_comparison_tool_reorg_tests=$enableval], [use_comparison_tool_reorg_tests=no]) +AC_ARG_ENABLE([extended-rpc-tests], + AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]), + [use_extended_rpc_tests=$enableval], + [use_extended_rpc_tests=no]) + AC_ARG_WITH([qrencode], [AS_HELP_STRING([--with-qrencode], [enable QR code support (default is yes if qt is enabled and libqrencode is found)])], @@ -342,6 +347,10 @@ else AC_SUBST(COMPARISON_TOOL_REORG_TESTS, 0) fi +if test x$use_extended_rpc_tests != xno; then + AC_SUBST(EXTENDED_RPC_TESTS, -extended) +fi + if test x$use_lcov = xyes; then if test x$LCOV = x; then AC_MSG_ERROR("lcov testing requested but lcov not found") From e3b5e6c39c5a6e56bdc21d307ac5cb6c80b737dd Mon Sep 17 00:00:00 2001 From: dexX7 Date: Mon, 12 Oct 2015 14:29:44 +0200 Subject: [PATCH 5/6] Run extended BitcoinJ tests for coverage based on config The configuration option `--enable-comparison-tool-reorg-tests` may be used to enable extended tests via BitcoinJ also for coverage testing. --- Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.am b/Makefile.am index 93a768b1c19..8b298619f72 100644 --- a/Makefile.am +++ b/Makefile.am @@ -170,7 +170,7 @@ test_bitcoin_filtered.info: test_bitcoin.info block_test.info: test_bitcoin_filtered.info $(MKDIR_P) qa/tmp - -@TIMEOUT=15 qa/pull-tester/run-bitcoind-for-test.sh $(JAVA) -jar $(JAVA_COMPARISON_TOOL) qa/tmp/compTool 0 + -@TIMEOUT=15 qa/pull-tester/run-bitcoind-for-test.sh $(JAVA) -jar $(JAVA_COMPARISON_TOOL) qa/tmp/compTool $(COMPARISON_TOOL_REORG_TESTS) $(LCOV) -c -d $(abs_builddir)/src --t BitcoinJBlockTest -o $@ $(LCOV) -z -d $(abs_builddir)/src $(LCOV) -z -d $(abs_builddir)/src/leveldb From d80e3cbece857b293a4903ef49c4d543bb2cfb7f Mon Sep 17 00:00:00 2001 From: dexX7 Date: Sun, 11 Oct 2015 18:29:42 +0200 Subject: [PATCH 6/6] Support gathering of code coverage data for RPC tests The RPC tests (via `qa/pull-tester/rpc-tests.py`) are now executed, when gathering code coverage data, for example with `make cov`. Generating coverage data requires `lcov`, which can installed with: sudo apt-get install lcov To also use the BitcoinJ tests, get the test tool: TOOL_URL=https://github.com/theuni/bitcoind-comparisontool/raw/master/pull-tests-8c6666f.jar TOOL_HASH=a865332b3827abcde684ab79f5f43c083b0b6a4c97ff5508c79f29fee24f11cd wget $TOOL_URL -O ./share/BitcoindComparisonTool.jar echo "$TOOL_HASH ./share/BitcoindComparisonTool.jar" | shasum --algorithm 256 --check The coverage data can be generated with: ./autogen.sh ./configure --enable-lcov --with-comparison-tool=./share/BitcoindComparisonTool.jar make make cov Optionally the options `--enable-extended-rpc-tests` and `--enable-comparison-tool-reorg-tests` may be used to enable more time consuming tests. It then runs the tests and generates two HTML reports: - test_bitcoin.coverage/index.html - total.coverage/index.html --- Makefile.am | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/Makefile.am b/Makefile.am index 8b298619f72..f0961c64ec0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -39,7 +39,7 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) $ COVERAGE_INFO = baseline_filtered_combined.info baseline.info block_test.info \ leveldb_baseline.info test_bitcoin_filtered.info total_coverage.info \ - baseline_filtered.info block_test_filtered.info \ + baseline_filtered.info block_test_filtered.info rpc_test.info rpc_test_filtered.info \ leveldb_baseline_filtered.info test_bitcoin_coverage.info test_bitcoin.info dist-hook: @@ -178,11 +178,20 @@ block_test.info: test_bitcoin_filtered.info block_test_filtered.info: block_test.info $(LCOV) -r $< "/usr/include/*" -o $@ +rpc_test.info: test_bitcoin_filtered.info + -@TIMEOUT=15 python qa/pull-tester/rpc-tests.py $(EXTENDED_RPC_TESTS) + $(LCOV) -c -d $(abs_builddir)/src --t rpc-tests -o $@ + $(LCOV) -z -d $(abs_builddir)/src + $(LCOV) -z -d $(abs_builddir)/src/leveldb + +rpc_test_filtered.info: rpc_test.info + $(LCOV) -r $< "/usr/include/*" -o $@ + test_bitcoin_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -o $@ -total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info block_test_filtered.info - $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a block_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt +total_coverage.info: baseline_filtered_combined.info test_bitcoin_filtered.info block_test_filtered.info rpc_test_filtered.info + $(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_bitcoin_filtered.info -a block_test_filtered.info -a rpc_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt test_bitcoin.coverage/.dirstamp: test_bitcoin_coverage.info $(GENHTML) -s $< -o $(@D)