1  #!/usr/bin/env bash
  2  #
  3  # Copyright (c) 2018-present The Bitcoin Core developers
  4  # Distributed under the MIT software license, see the accompanying
  5  # file COPYING or http://www.opensource.org/licenses/mit-license.php.
  6  
  7  export LC_ALL=C.UTF-8
  8  
  9  set -ex
 10  
 11  export ASAN_OPTIONS="detect_leaks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
 12  export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 13  export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:second_deadlock_stack=1"
 14  export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
 15  
 16  echo "Number of available processing units: $(nproc)"
 17  if [ "$CI_OS_NAME" == "macos" ]; then
 18    top -l 1 -s 0 | awk ' /PhysMem/ {print}'
 19  else
 20    free -m -h
 21    echo "System info: $(uname --kernel-name --kernel-release)"
 22    lscpu
 23  fi
 24  echo "Free disk space:"
 25  df -h
 26  
 27  # We force an install of linux-headers again here via $PACKAGES to fix any
 28  # kernel mismatch between a cached docker image and the underlying host.
 29  # This can happen occasionally on hosted runners if the runner image is updated.
 30  if [[ "$CONTAINER_NAME" == "ci_native_asan" ]]; then
 31    $CI_RETRY_EXE apt-get update
 32    ${CI_RETRY_EXE} bash -c "apt-get install --no-install-recommends --no-upgrade -y $PACKAGES"
 33  fi
 34  
 35  # What host to compile for. See also ./depends/README.md
 36  # Tests that need cross-compilation export the appropriate HOST.
 37  # Tests that run natively guess the host
 38  export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
 39  
 40  echo "=== BEGIN env ==="
 41  env
 42  echo "=== END env ==="
 43  
 44  # Don't apply patches in the tidy job, because it relies on the `git diff`
 45  # command to detect IWYU errors. It is safe to skip this patch in the tidy job
 46  # because it doesn't run a UB detector.
 47  if [ "$RUN_TIDY" != "true" ]; then
 48    # compact->outputs[i].file_size is uninitialized memory, so reading it is UB.
 49    # The statistic bytes_written is only used for logging, which is disabled in
 50    # CI, so as a temporary minimal fix to work around UB and CI failures, leave
 51    # bytes_written unmodified.
 52    # See https://github.com/bitcoin/bitcoin/pull/28359#issuecomment-1698694748
 53    # Tee patch to stdout to make it clear CI is testing modified code.
 54    tee >(patch -p1) <<'EOF'
 55  --- a/src/leveldb/db/db_impl.cc
 56  +++ b/src/leveldb/db/db_impl.cc
 57  @@ -1028,9 +1028,6 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
 58         stats.bytes_read += compact->compaction->input(which, i)->file_size;
 59       }
 60     }
 61  -  for (size_t i = 0; i < compact->outputs.size(); i++) {
 62  -    stats.bytes_written += compact->outputs[i].file_size;
 63  -  }
 64  
 65     mutex_.Lock();
 66     stats_[compact->compaction->level() + 1].Add(stats);
 67  EOF
 68  fi
 69  
 70  if [ "$RUN_FUZZ_TESTS" = "true" ]; then
 71    export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_corpora/
 72    if [ ! -d "$DIR_FUZZ_IN" ]; then
 73      ${CI_RETRY_EXE} git clone --depth=1 https://github.com/bitcoin-core/qa-assets "${DIR_QA_ASSETS}"
 74    fi
 75    (
 76      cd "${DIR_QA_ASSETS}"
 77      echo "Using qa-assets repo from commit ..."
 78      git log -1
 79    )
 80  elif [ "$RUN_UNIT_TESTS" = "true" ]; then
 81    export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/
 82    if [ ! -d "$DIR_UNIT_TEST_DATA" ]; then
 83      mkdir -p "$DIR_UNIT_TEST_DATA"
 84      ${CI_RETRY_EXE} curl --location --fail https://github.com/bitcoin-core/qa-assets/raw/main/unit_test_data/script_assets_test.json -o "${DIR_UNIT_TEST_DATA}/script_assets_test.json"
 85    fi
 86  fi
 87  
 88  # Make sure default datadir does not exist and is never read by creating a dummy file
 89  if [ "$CI_OS_NAME" == "macos" ]; then
 90    echo > "${HOME}/Library/Application Support/Bitcoin"
 91  else
 92    echo > "${HOME}/.bitcoin"
 93  fi
 94  
 95  if [ -z "$NO_DEPENDS" ]; then
 96    if [[ $CI_IMAGE_NAME_TAG == *alpine* ]]; then
 97      SHELL_OPTS="CONFIG_SHELL=/usr/bin/dash"
 98    else
 99      SHELL_OPTS="CONFIG_SHELL="
100    fi
101    bash -c "$SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS LOG=1"
102  fi
103  if [ "$DOWNLOAD_PREVIOUS_RELEASES" = "true" ]; then
104    test/get_previous_releases.py --target-dir "$PREVIOUS_RELEASES_DIR"
105  fi
106  
107  BITCOIN_CONFIG_ALL="-DBUILD_BENCH=ON -DBUILD_FUZZ_BINARY=ON"
108  if [ -z "$NO_DEPENDS" ]; then
109    BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} -DCMAKE_TOOLCHAIN_FILE=$DEPENDS_DIR/$HOST/toolchain.cmake"
110  fi
111  if [ -z "$NO_WERROR" ]; then
112    BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} -DWERROR=ON"
113  fi
114  
115  ccache --zero-stats
116  PRINT_CCACHE_STATISTICS="ccache --version | head -n 1 && ccache --show-stats"
117  
118  # Folder where the build is done.
119  BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build-$HOST}
120  
121  BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_INSTALL_PREFIX=$BASE_OUTDIR -Werror=dev"
122  
123  if [[ "${RUN_TIDY}" == "true" ]]; then
124    BITCOIN_CONFIG_ALL="$BITCOIN_CONFIG_ALL -DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
125  fi
126  
127  eval "CMAKE_ARGS=($BITCOIN_CONFIG_ALL $BITCOIN_CONFIG)"
128  cmake -S "$BASE_ROOT_DIR" -B "$BASE_BUILD_DIR" "${CMAKE_ARGS[@]}" || (
129    cd "${BASE_BUILD_DIR}"
130    # shellcheck disable=SC2046
131    cat $(cmake -P "${BASE_ROOT_DIR}/ci/test/GetCMakeLogFiles.cmake")
132    false
133  )
134  
135  # shellcheck disable=SC2086
136  cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target all $GOAL || (
137    echo "Build failure. Verbose build follows."
138    # shellcheck disable=SC2086
139    cmake --build "${BASE_BUILD_DIR}" -j1 --target all $GOAL --verbose
140    false
141  )
142  
143  bash -c "${PRINT_CCACHE_STATISTICS}"
144  if [ "$CI" = "true" ]; then
145    hit_rate=$(ccache -s | grep "Hits:" | head -1 | sed 's/.*(\(.*\)%).*/\1/')
146    if [ "${hit_rate%.*}" -lt 75 ]; then
147        echo "::notice title=low ccache hitrate::Ccache hit-rate in $CONTAINER_NAME was $hit_rate%"
148    fi
149  fi
150  du -sh "${DEPENDS_DIR}"/*/
151  du -sh "${PREVIOUS_RELEASES_DIR}"
152  
153  if [ -n "${CI_LIMIT_STACK_SIZE}" ]; then
154    ulimit -s 512
155  fi
156  
157  if [ -n "$USE_VALGRIND" ]; then
158    "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
159  fi
160  
161  if [ "$RUN_CHECK_DEPS" = "true" ]; then
162    "${BASE_ROOT_DIR}/contrib/devtools/check-deps.sh" "${BASE_BUILD_DIR}"
163  fi
164  
165  if [ "$RUN_UNIT_TESTS" = "true" ]; then
166    DIR_UNIT_TEST_DATA="${DIR_UNIT_TEST_DATA}" \
167    LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \
168    CTEST_OUTPUT_ON_FAILURE=ON \
169    ctest --test-dir "${BASE_BUILD_DIR}" \
170      --stop-on-failure \
171      "${MAKEJOBS}" \
172      --timeout $(( TEST_RUNNER_TIMEOUT_FACTOR * 60 ))
173  fi
174  
175  if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then
176    # parses TEST_RUNNER_EXTRA as an array which allows for multiple arguments such as TEST_RUNNER_EXTRA='--exclude "rpc_bind.py --ipv6"'
177    eval "TEST_RUNNER_EXTRA=($TEST_RUNNER_EXTRA)"
178    LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \
179    "${BASE_BUILD_DIR}/test/functional/test_runner.py" \
180      --ci "${MAKEJOBS}" \
181      --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" \
182      --ansi \
183      --combinedlogslen=99999999 \
184      --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" \
185      "${TEST_RUNNER_EXTRA[@]}" \
186      --quiet \
187      --failfast
188  fi
189  
190  if [ "${RUN_TIDY}" = "true" ]; then
191    cmake -B /tidy-build -DLLVM_DIR=/usr/lib/llvm-"${TIDY_LLVM_V}"/cmake -DCMAKE_BUILD_TYPE=Release -S "${BASE_ROOT_DIR}"/contrib/devtools/bitcoin-tidy
192    cmake --build /tidy-build "$MAKEJOBS"
193    cmake --build /tidy-build --target bitcoin-tidy-tests "$MAKEJOBS"
194  
195    set -eo pipefail
196    # Filter out:
197    # * qt qrc and moc generated files
198    jq 'map(select(.file | test("src/qt/.*_autogen/.*\\.cpp$") | not))' "${BASE_BUILD_DIR}/compile_commands.json" > tmp.json
199    mv tmp.json "${BASE_BUILD_DIR}/compile_commands.json"
200  
201    cd "${BASE_BUILD_DIR}/src/"
202    if ! ( run-clang-tidy-"${TIDY_LLVM_V}" -quiet -load="/tidy-build/libbitcoin-tidy.so" "${MAKEJOBS}" | tee tmp.tidy-out.txt ); then
203      grep -C5 "error: " tmp.tidy-out.txt
204      echo "^^^ ⚠️ Failure generated from clang-tidy"
205      false
206    fi
207  
208    # TODO: Consider enforcing IWYU across the entire codebase.
209    FILES_WITH_ENFORCED_IWYU="/src/(crypto|index)/.*\\.cpp"
210    jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json"
211    jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns) | not))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_warnings.json"
212  
213    cd "${BASE_ROOT_DIR}"
214  
215    run_iwyu() {
216      mv "${BASE_BUILD_DIR}/$1" "${BASE_BUILD_DIR}/compile_commands.json"
217      python3 "/include-what-you-use/iwyu_tool.py" \
218               -p "${BASE_BUILD_DIR}" "${MAKEJOBS}" \
219               -- -Xiwyu --cxx17ns -Xiwyu --mapping_file="${BASE_ROOT_DIR}/contrib/devtools/iwyu/bitcoin.core.imp" \
220               -Xiwyu --max_line_length=160 \
221               2>&1 | tee /tmp/iwyu_ci.out
222      python3 "/include-what-you-use/fix_includes.py" --nosafe_headers < /tmp/iwyu_ci.out
223    }
224  
225    run_iwyu "compile_commands_iwyu_errors.json"
226    if ! ( git --no-pager diff --exit-code ); then
227      echo "^^^ ⚠️ Failure generated from IWYU"
228      false
229    fi
230  
231    run_iwyu "compile_commands_iwyu_warnings.json"
232    git --no-pager diff
233  fi
234  
235  if [ "$RUN_FUZZ_TESTS" = "true" ]; then
236    # shellcheck disable=SC2086
237    LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" \
238    "${BASE_BUILD_DIR}/test/fuzz/test_runner.py" \
239      ${FUZZ_TESTS_CONFIG} \
240      "${MAKEJOBS}" \
241      -l DEBUG \
242      "${DIR_FUZZ_IN}" \
243      --empty_min_time=60
244  fi