diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 6390667394f..e9c54139a70 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -74,39 +74,3 @@ CI_EXEC rsync --archive --stats --human-readable /ro_base/ "${BASE_ROOT_DIR}" || CI_EXEC git config --global --add safe.directory \"*\" CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" - -if [ "$CI_OS_NAME" == "macos" ]; then - top -l 1 -s 0 | awk ' /PhysMem/ {print}' - echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" -else - CI_EXEC free -m -h - CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) - CI_EXEC echo "$(lscpu | grep Endian)" -fi -CI_EXEC echo "Free disk space:" -CI_EXEC df -h - -if [ "$RUN_FUZZ_TESTS" = "true" ]; then - export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ - if [ ! -d "$DIR_FUZZ_IN" ]; then - CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" - fi -elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ - if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then - CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" - CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" - fi -fi - -CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" - -if [ "$USE_BUSY_BOX" = "true" ]; then - echo "Setup to use BusyBox utils" - # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) - # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done - # Print BusyBox version - CI_EXEC patch --help -fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh deleted file mode 100755 index 199cdd64a7c..00000000000 --- a/ci/test/05_before_script.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -# Make sure default datadir does not exist and is never read by creating a dummy file -if [ "$CI_OS_NAME" == "macos" ]; then - echo > "${HOME}/Library/Application Support/Bitcoin" -else - CI_EXEC echo \> \$HOME/.bitcoin -fi - -if [ -z "$NO_DEPENDS" ]; then - if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then - # CentOS has problems building the depends if the config shell is not explicitly set - # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to - # an error as the first command is executed) - SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" - else - SHELL_OPTS="CONFIG_SHELL=" - fi - CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 -fi -if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then - CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" -fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh deleted file mode 100755 index 5856d33d2db..00000000000 --- a/ci/test/06_script_a.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2022 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" -if [ -z "$NO_DEPENDS" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" -fi -if [ -z "$NO_WERROR" ]; then - BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" -fi - -CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" -PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" - -if [ -n "$ANDROID_TOOLS_URL" ]; then - CI_EXEC make distclean || true - CI_EXEC ./autogen.sh - CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" - CI_EXEC "${PRINT_CCACHE_STATISTICS}" - exit 0 -fi - -BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" - -if [ -n "$CONFIG_SHELL" ]; then - CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" -else - CI_EXEC ./autogen.sh -fi - -CI_EXEC mkdir -p "${BASE_BUILD_DIR}" -export P_CI_DIR="${BASE_BUILD_DIR}" - -CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -CI_EXEC make distdir VERSION="$HOST" - -export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" - -CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) - -set -o errtrace -trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR - -if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - # MemorySanitizer (MSAN) does not support tracking memory initialization done by - # using the Linux getrandom syscall. Avoid using getrandom by undefining - # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for - # details. - CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' -fi - -if [[ "${RUN_TIDY}" == "true" ]]; then - MAYBE_BEAR="bear --config src/.bear-tidy-config" - MAYBE_TOKEN="--" -fi - -CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) - -CI_EXEC "${PRINT_CCACHE_STATISTICS}" -CI_EXEC du -sh "${DEPENDS_DIR}"/*/ -CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 3efbe8afc6c..fa6dced5209 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -8,6 +8,125 @@ export LC_ALL=C.UTF-8 set -ex +if [ "$CI_OS_NAME" == "macos" ]; then + top -l 1 -s 0 | awk ' /PhysMem/ {print}' + echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" +else + CI_EXEC free -m -h + CI_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) + CI_EXEC echo "$(lscpu | grep Endian)" +fi +CI_EXEC echo "Free disk space:" +CI_EXEC df -h + +if [ "$RUN_FUZZ_TESTS" = "true" ]; then + export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + if [ ! -d "$DIR_FUZZ_IN" ]; then + CI_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}" + fi +elif [ "$RUN_UNIT_TESTS" = "true" ] || [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then + export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ + if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then + CI_EXEC mkdir -p "$DIR_UNIT_TEST_DATA" + CI_EXEC curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json" + fi +fi + +CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" + +if [ "$USE_BUSY_BOX" = "true" ]; then + echo "Setup to use BusyBox utils" + # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) + # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) + # shellcheck disable=SC1010 + CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done + # Print BusyBox version + CI_EXEC patch --help +fi + +# Make sure default datadir does not exist and is never read by creating a dummy file +if [ "$CI_OS_NAME" == "macos" ]; then + echo > "${HOME}/Library/Application Support/Bitcoin" +else + CI_EXEC echo \> \$HOME/.bitcoin +fi + +if [ -z "$NO_DEPENDS" ]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + # CentOS has problems building the depends if the config shell is not explicitly set + # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to + # an error as the first command is executed) + SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/dash" + else + SHELL_OPTS="CONFIG_SHELL=" + fi + CI_EXEC "$SHELL_OPTS" make "$MAKEJOBS" -C depends HOST="$HOST" "$DEP_OPTS" LOG=1 +fi +if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then + CI_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" +fi + +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi + +CI_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" +PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats" + +if [ -n "$ANDROID_TOOLS_URL" ]; then + CI_EXEC make distclean || true + CI_EXEC ./autogen.sh + CI_EXEC ./configure "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) + CI_EXEC "make $MAKEJOBS && cd src/qt && ANDROID_HOME=${ANDROID_HOME} ANDROID_NDK_HOME=${ANDROID_NDK_HOME} make apk" + CI_EXEC "${PRINT_CCACHE_STATISTICS}" + exit 0 +fi + +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" + +if [ -n "$CONFIG_SHELL" ]; then + CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" +else + CI_EXEC ./autogen.sh +fi + +CI_EXEC mkdir -p "${BASE_BUILD_DIR}" +export P_CI_DIR="${BASE_BUILD_DIR}" + +CI_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) + +CI_EXEC make distdir VERSION="$HOST" + +export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" + +CI_EXEC ./configure --cache-file=../config.cache "$BITCOIN_CONFIG_ALL" "$BITCOIN_CONFIG" || ( (CI_EXEC cat config.log) && false) + +set -o errtrace +trap 'CI_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR + +if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then + # MemorySanitizer (MSAN) does not support tracking memory initialization done by + # using the Linux getrandom syscall. Avoid using getrandom by undefining + # HAVE_SYS_GETRANDOM. See https://github.com/google/sanitizers/issues/852 for + # details. + CI_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' +fi + +if [[ "${RUN_TIDY}" == "true" ]]; then + MAYBE_BEAR="bear --config src/.bear-tidy-config" + MAYBE_TOKEN="--" +fi + +CI_EXEC "${MAYBE_BEAR}" "${MAYBE_TOKEN}" make "$MAKEJOBS" "$GOAL" || ( echo "Build failure. Verbose build follows." && CI_EXEC make "$GOAL" V=1 ; false ) + +CI_EXEC "${PRINT_CCACHE_STATISTICS}" +CI_EXEC du -sh "${DEPENDS_DIR}"/*/ +CI_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" + if [[ $HOST = *-mingw32 ]]; then # Generate all binaries, so that they can be wrapped make "$MAKEJOBS" -C src/secp256k1 VERBOSE=1 diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh index 751a4056df7..2284a2903b7 100755 --- a/ci/test_run_all.sh +++ b/ci/test_run_all.sh @@ -8,8 +8,6 @@ export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh set -o errexit; source ./ci/test/04_install.sh -set -o errexit; source ./ci/test/05_before_script.sh -set -o errexit; source ./ci/test/06_script_a.sh set -o errexit CI_EXEC "${BASE_ROOT_DIR}/ci/test/06_script_b.sh"