diff options
author | Elliott Hughes <enh@google.com> | 2021-04-02 19:51:01 +0000 |
---|---|---|
committer | Automerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com> | 2021-04-02 19:51:01 +0000 |
commit | 75c8dcf71ca8652f671b4ca5fea780a558c86e08 (patch) | |
tree | 328e6c9629b196cec1de3a94ee804d9fee3a0524 /infra | |
parent | 378a8d19d33a5a62afbbe33b7f7b87b67db47236 (diff) | |
parent | 235e96b2f8ab4e43316158a2e6fa69e75a219e23 (diff) | |
download | oss-fuzz-75c8dcf71ca8652f671b4ca5fea780a558c86e08.tar.gz |
Upgrade oss-fuzz to 947169dc86572e121c3e138f366a9f39ac6266ae am: f3764d0712 am: 1117028736 am: 235e96b2f8android-12.1.0_r9android-12.1.0_r8android-12.1.0_r7android-12.1.0_r26android-12.1.0_r25android-12.1.0_r24android-12.1.0_r23android-12.1.0_r22android-12.1.0_r21android-12.1.0_r20android-12.1.0_r19android-12.1.0_r18android-12.1.0_r17android-12.1.0_r16android-12.1.0_r15android-12.1.0_r14android-12.1.0_r13android-12.1.0_r12android-12.1.0_r11android-12.1.0_r10android-12.0.0_r32android-12.0.0_r29android-12.0.0_r28android-12.0.0_r27android-12.0.0_r26android-12.0.0_r21android-12.0.0_r20android-12.0.0_r19android-12.0.0_r18android-12.0.0_r16android12L-devandroid12L-d2-s8-releaseandroid12L-d2-s7-releaseandroid12L-d2-s6-releaseandroid12L-d2-s5-releaseandroid12L-d2-s4-releaseandroid12L-d2-s3-releaseandroid12L-d2-s2-releaseandroid12L-d2-s1-releaseandroid12L-d2-releaseandroid12-qpr3-s7-releaseandroid12-qpr3-s6-releaseandroid12-qpr3-s5-releaseandroid12-qpr3-s4-releaseandroid12-qpr3-s3-releaseandroid12-qpr3-s2-releaseandroid12-qpr3-s1-releaseandroid12-qpr3-releaseandroid12-qpr1-releaseandroid12-qpr1-d-s3-releaseandroid12-qpr1-d-s2-releaseandroid12-qpr1-d-s1-releaseandroid12-qpr1-d-releaseandroid12-dev
Original change: https://android-review.googlesource.com/c/platform/external/oss-fuzz/+/1662261
Change-Id: Ib7b7a79b38e1261c1d5fb4ccb1a5dfd106588996
Diffstat (limited to 'infra')
95 files changed, 1411 insertions, 595 deletions
diff --git a/infra/.dockerignore b/infra/.dockerignore index 1e82e3e75..c78653342 100644 --- a/infra/.dockerignore +++ b/infra/.dockerignore @@ -1 +1,9 @@ -test_files
\ No newline at end of file +cifuzz/test_data/* + +# Copied from .gitignore. +.vscode/ +*.pyc +build +*~ +.DS_Store +*.swp
\ No newline at end of file diff --git a/infra/.pylintrc b/infra/.pylintrc deleted file mode 100644 index 8ce0b2226..000000000 --- a/infra/.pylintrc +++ /dev/null @@ -1,2 +0,0 @@ -[FORMAT] -indent-string = " "
\ No newline at end of file diff --git a/infra/base-images/Jenkinsfile b/infra/base-images/Jenkinsfile deleted file mode 100644 index 67b22e9b0..000000000 --- a/infra/base-images/Jenkinsfile +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -// Jenkins build script for base images. -node { - git url: 'https://github.com/google/oss-fuzz/' - - stage("infra/base-images/all.sh") { - sh "infra/base-images/all.sh --no-cache" - } - - stage("docker push") { - def images = ['ossfuzz/base-image', 'ossfuzz/base-clang', 'ossfuzz/base-libfuzzer', - 'ossfuzz/base-runner', 'ossfuzz/base-runner-debug', - 'ossfuzz/base-builder',] - - docker.withRegistry('', 'docker-login') { - for (int i = 0; i < images.size(); i++) { - def image = images[i] - docker.image(image).push() - } - } - } -} diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 6f596d5ba..d802f247a 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -29,6 +29,7 @@ RUN dpkg --add-architecture i386 && \ jq \ libc6-dev-i386 \ patchelf \ + rsync \ subversion \ zip @@ -57,12 +58,14 @@ RUN export PYTHON_DEPS="\ ln -s /usr/bin/python3 /usr/bin/python && \ cd .. && \ rm -r /tmp/Python-$PYTHON_VERSION.tar.xz /tmp/Python-$PYTHON_VERSION && \ - apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888 + rm -rf /usr/local/lib/python3.8/test && \ + apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888 # Install latest atheris for python fuzzing, pyinstaller for fuzzer packaging, # six for Bazel rules. -RUN unset CFLAGS CXXFLAGS && pip3 install -v \ - atheris pyinstaller==4.1 six==1.15.0 +RUN unset CFLAGS CXXFLAGS && pip3 install -v --no-cache-dir \ + atheris pyinstaller==4.1 six==1.15.0 && \ + rm -rf /tmp/* # Download and install the latest stable Go. RUN cd /tmp && \ @@ -87,15 +90,42 @@ ENV CARGO_HOME=/rust ENV RUSTUP_HOME=/rust/rustup ENV PATH=$PATH:/rust/bin RUN curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal -RUN cargo install cargo-fuzz +RUN cargo install cargo-fuzz && rm -rf /rust/registry # Needed to recompile rust std library for MSAN RUN rustup component add rust-src --toolchain nightly +# Set up custom environment variable for source code copy for coverage reports +ENV OSSFUZZ_RUSTPATH /rust # Install Bazel through Bazelisk, which automatically fetches the latest Bazel version. ENV BAZELISK_VERSION 1.7.4 RUN curl -L https://github.com/bazelbuild/bazelisk/releases/download/v$BAZELISK_VERSION/bazelisk-linux-amd64 -o /usr/local/bin/bazel && \ chmod +x /usr/local/bin/bazel +# Install OpenJDK 15 and trim its size by removing unused components. +ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64 +ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server +ENV PATH=$PATH:$JAVA_HOME/bin +RUN cd /tmp && \ + curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \ + mkdir -p $JAVA_HOME && \ + tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ + rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \ + rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip + +# Install the latest Jazzer in $OUT. +# jazzer_api_deploy.jar is required only at build-time, the agent and the +# drivers are copied to $OUT as they need to be present on the runners. +ENV JAZZER_API_PATH "/usr/local/lib/jazzer_api_deploy.jar" +RUN cd $SRC/ && \ + git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \ + cd jazzer && \ + bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \ + //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //agent:jazzer_api_deploy.jar && \ + cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan /usr/local/bin/ && \ + cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH && \ + rm -rf ~/.cache/bazel ~/.cache/bazelisk && \ + rm -rf $SRC/jazzer + # Default build flags for various sanitizers. ENV SANITIZER_FLAGS_address "-fsanitize=address -fsanitize-address-use-after-scope" @@ -106,6 +136,8 @@ ENV SANITIZER_FLAGS_memory "-fsanitize=memory -fsanitize-memory-track-origins" ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow" +ENV SANITIZER_FLAGS_thread "-fsanitize=thread" + # Do not use any sanitizers in the coverage build. ENV SANITIZER_FLAGS_coverage "" @@ -144,15 +176,12 @@ ENV LIB_FUZZING_ENGINE="/usr/lib/libFuzzingEngine.a" # TODO: remove after tpm2 catchup. ENV FUZZER_LDFLAGS "" -ENV PRECOMPILED_DIR="/usr/lib/precompiled" -RUN mkdir $PRECOMPILED_DIR - WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout aeb7d7048371cd91ab9280c3958f1c35e5d5e758 + git checkout 2102264acf5c271b7560a82771b3af8136af9354 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ @@ -161,12 +190,14 @@ RUN cd $SRC && \ tar -xzv --strip-components=1 -f $SRC/oss-fuzz.tar.gz && \ rm -rf examples $SRC/oss-fuzz.tar.gz -COPY compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ - compile_go_fuzzer precompile_honggfuzz srcmap write_labels.py /usr/local/bin/ +COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ + compile_go_fuzzer precompile_honggfuzz precompile_afl debug_afl srcmap \ + write_labels.py bazel_build_fuzz_tests /usr/local/bin/ COPY detect_repo.py /opt/cifuzz/ COPY ossfuzz_coverage_runner.go $GOPATH RUN precompile_honggfuzz +RUN precompile_afl CMD ["compile"] diff --git a/infra/base-images/base-builder/bazel_build_fuzz_tests b/infra/base-images/base-builder/bazel_build_fuzz_tests new file mode 100755 index 000000000..86740ee01 --- /dev/null +++ b/infra/base-images/base-builder/bazel_build_fuzz_tests @@ -0,0 +1,80 @@ +#!/bin/bash -eu +# +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +: "${BAZEL_FUZZ_TEST_TAG:=fuzz-test}" +: "${BAZEL_FUZZ_TEST_EXCLUDE_TAG:=no-oss-fuzz}" +: "${BAZEL_PACKAGE_SUFFIX:=_oss_fuzz}" +: "${BAZEL_TOOL:=bazel}" +: "${BAZEL_EXTRA_BUILD_FLAGS:=}" + +if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then + BAZEL_FUZZ_TEST_QUERY=" + let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in + \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests) + " +fi + +echo "Using Bazel query to find fuzz targets: ${BAZEL_FUZZ_TEST_QUERY}" + +declare -r OSS_FUZZ_TESTS=( + $(bazel query "${BAZEL_FUZZ_TEST_QUERY}" | sed "s/$/${BAZEL_PACKAGE_SUFFIX}/") +) + +echo "Found ${#OSS_FUZZ_TESTS[@]} fuzz test packages:" +for oss_fuzz_test in "${OSS_FUZZ_TESTS[@]}"; do + echo " ${oss_fuzz_test}" +done + +declare -r BAZEL_BUILD_FLAGS=( + "-c" "opt" + "--//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \ + "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \ + "--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \ + "--linkopt=-lc++" \ + "--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \ + ${BAZEL_EXTRA_BUILD_FLAGS[*]} +) + +echo "Building the fuzz tests with the following Bazel options:" +echo " ${BAZEL_BUILD_FLAGS[@]}" + +${BAZEL_TOOL} build "${BAZEL_BUILD_FLAGS[@]}" "${OSS_FUZZ_TESTS[@]}" + +echo "Extracting the fuzz test packages in the output directory." +for oss_fuzz_archive in $(find bazel-bin/ -name "*${BAZEL_PACKAGE_SUFFIX}.tar"); do + tar -xvf "${oss_fuzz_archive}" -C "${OUT}" +done + +if [ "$SANITIZER" = "coverage" ]; then + echo "Collecting the repository source files for coverage tracking." + declare -r COVERAGE_SOURCES="${OUT}/proc/self/cwd" + mkdir -p "${COVERAGE_SOURCES}" + declare -r RSYNC_FILTER_ARGS=( + "--include" "*.h" + "--include" "*.cc" + "--include" "*.hpp" + "--include" "*.cpp" + "--include" "*.c" + "--include" "*.inc" + "--include" "*/" + "--exclude" "*" + ) + rsync -avLk "${RSYNC_FILTER_ARGS[@]}" \ + "$(bazel info execution_root)/" \ + "${COVERAGE_SOURCES}/" +fi diff --git a/infra/base-images/base-builder/cargo b/infra/base-images/base-builder/cargo new file mode 100755 index 000000000..bed8e7660 --- /dev/null +++ b/infra/base-images/base-builder/cargo @@ -0,0 +1,51 @@ +#!/bin/bash -eu +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is a wrapper around calling cargo +# This just expands RUSTFLAGS in case of a coverage build +# We need this until https://github.com/rust-lang/cargo/issues/5450 is merged +# because cargo uses relative paths for the current crate +# and absolute paths for its dependencies +# +################################################################################ + +if [ "$SANITIZER" = "coverage" ] && [ $1 = "build" ] +then + crate_src_abspath=`cargo metadata --no-deps --format-version 1 | jq -r '.workspace_root'` + export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix src=$crate_src_abspath/src" +fi + +if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ] +then + # hack to turn cargo fuzz build into cargo build so as to get coverage + # cargo fuzz adds "--target" "x86_64-unknown-linux-gnu" + ( + # go into fuzz directory if not already the case + cd fuzz || true + fuzz_src_abspath=`pwd` + export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix fuzz_targets=$fuzz_src_abspath/fuzz_targets" + # we do not want to trigger debug assertions and stops + export RUSTFLAGS="$RUSTFLAGS -C debug-assertions=no" + # do not optimize with --release, leading to Malformed instrumentation profile data + cargo build --bins + # copies the build output in the expected target directory + cd `cargo metadata --format-version 1 --no-deps | jq -r '.target_directory'` + mkdir -p x86_64-unknown-linux-gnu/release + cp -r debug/* x86_64-unknown-linux-gnu/release/ + ) + exit 0 +fi + +/rust/bin/cargo "$@" diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index 2bf20b1e3..78453c98c 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -22,6 +22,21 @@ if [ "$SANITIZER" = "dataflow" ] && [ "$FUZZING_ENGINE" != "dataflow" ]; then exit 1 fi +if [ "$FUZZING_LANGUAGE" = "jvm" ]; then + if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then + echo "ERROR: JVM projects can be fuzzed with libFuzzer engine only." + exit 1 + fi + if [ "$SANITIZER" != "address" ]; then + echo "ERROR: JVM projects can be fuzzed with AddressSanitizer only." + exit 1 + fi + if [ "$ARCHITECTURE" != "x86_64" ]; then + echo "ERROR: JVM projects can be fuzzed on x86_64 architecture only." + exit 1 + fi +fi + if [ "$FUZZING_LANGUAGE" = "python" ]; then if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then echo "ERROR: Python projects can be fuzzed with libFuzzer engine only." @@ -46,7 +61,8 @@ if [[ $ARCHITECTURE == "i386" ]]; then export CFLAGS="-m32 $CFLAGS" cp -R /usr/i386/lib/* /usr/lib fi -if [[ $FUZZING_ENGINE != "none" ]]; then +# JVM projects are fuzzed with Jazzer, which has libFuzzer built in. +if [[ $FUZZING_ENGINE != "none" ]] && [[ $FUZZING_LANGUAGE != "jvm" ]]; then # compile script might override environment, use . to call it. . compile_${FUZZING_ENGINE} fi @@ -87,6 +103,11 @@ if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$ARCH else export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers" fi +if [ "$SANITIZER" = "coverage" ] +then + # link to C++ from comment in f5098035eb1a14aa966c8651d88ea3d64323823d + export RUSTFLAGS="$RUSTFLAGS -Zinstrument-coverage -C link-arg=-lc++" +fi # Add Rust libfuzzer flags. # See https://github.com/rust-fuzz/libfuzzer/blob/master/build.rs#L12. @@ -113,6 +134,11 @@ fi # Copy latest llvm-symbolizer in $OUT for stack symbolization. cp $(which llvm-symbolizer) $OUT/ +# Copy Jazzer to $OUT if needed. +if [ "$FUZZING_LANGUAGE" = "jvm" ]; then + cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $(which jazzer_driver_asan) $OUT/ +fi + echo "---------------------------------------------------------------" echo "CC=$CC" echo "CXX=$CXX" @@ -124,7 +150,7 @@ BUILD_CMD="bash -eux $SRC/build.sh" # We need to preserve source code files for generating a code coverage report. # We need exact files that were compiled, so copy both $SRC and $WORK dirs. -COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OUT" +COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH $OUT" if [ "${BUILD_UID-0}" -ne "0" ]; then adduser -u $BUILD_UID --disabled-password --gecos '' builder diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl index 318eca44e..dc6624459 100644 --- a/infra/base-images/base-builder/compile_afl +++ b/infra/base-images/base-builder/compile_afl @@ -15,43 +15,78 @@ # ################################################################################ -echo "Compiling afl++" +# afl++ configuration options. +# The 'env|grep' setup ensures we do not trigger the linter. +# The variables need to be set to "1" here - or before running this script. -# Build and copy afl++ tools necessary for fuzzing. +# AFL++ settings. +export AFL_LLVM_MODE_WORKAROUND=0 +export AFL_ENABLE_DICTIONARY=0 + +# Start compiling afl++. +echo "Copying precompiled afl++" + +# Copy afl++ tools necessary for fuzzing. pushd $SRC/aflplusplus > /dev/null -# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it -# down with sanitizers. -INITIAL_CXXFLAGS=$CXXFLAGS -INITIAL_CFLAGS=$CFLAGS -unset CXXFLAGS -unset CFLAGS -make clean -AFL_NO_X86=1 PYTHON_INCLUDE=/ make -CFLAGS=$INITIAL_CFLAGS -CXXFLAGS=$INITIAL_CXXFLAGS - -# Build afl++ driver with existing CFLAGS, CXXFLAGS. -make -C utils/aflpp_driver -cp libAFLDriver.a $LIB_FUZZING_ENGINE +cp -f libAFLDriver.a $LIB_FUZZING_ENGINE # Some important projects include libraries, copy those even when they don't # start with "afl-". Use "sort -u" to avoid a warning about duplicates. ls afl-* *.txt *.a *.o *.so | sort -u | xargs cp -t $OUT -popd > /dev/null - export CC="$SRC/aflplusplus/afl-clang-fast" export CXX="$SRC/aflplusplus/afl-clang-fast++" # Set sane afl++ environment defaults: # Be quiet, otherwise this can break some builds. export AFL_QUIET=1 -# Several targets run their own tools, so ensure its working. -export AFL_MAP_SIZE=4194304 # No leak errors during builds. -export ASAN_OPTIONS="detect_leaks=0:symbolize=0" -# -# Placeholder for the upcoming afl++ build options roulette -# +export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1" + +# AFL compile option roulette. It is OK if they all happen together. + +# 40% chance to perform CMPLOG +rm -f "$OUT/afl_cmplog.txt" +test $(($RANDOM % 10)) -lt 4 && { + export AFL_LLVM_CMPLOG=1 + touch "$OUT/afl_cmplog.txt" +} + +# 10% chance to perform LAF_INTEL +test $(($RANDOM % 10)) -lt 1 && { + export AFL_LLVM_LAF_ALL=1 +} + +# If the targets wants a dictionary - then create one. +test "$AFL_ENABLE_DICTIONARY" = "1" && { + export AFL_LLVM_DICT2FILE="$OUT/afl++.dict" +} + +# In case afl-clang-fast ever breaks, this is a workaround: +test "$AFL_LLVM_MODE_WORKAROUND" = "1" && { + export CC=clang + export CXX=clang++ + WORKAROUND_FLAGS=-fsanitize-coverage=trace-pc-guard + # We can still do CMPLOG light: + test -e "$OUT/afl_cmplog.txt" && { + WORKAROUND_FLAGS="$WORKAROUND_FLAGS",trace-cmp + } + export CFLAGS="$CFLAGS $WORKAROUND_FLAGS" + export CXXFLAGS="$CXXFLAGS $WORKAROUND_FLAGS" + unset AFL_LLVM_LAF_ALL + unset AFL_LLVM_DICT2FILE + unset AFL_ENABLE_DICTIONARY + # We need to create a new fuzzer lib however. + ar ru libAFLDrivernew.a afl-compiler-rt.o utils/aflpp_driver/aflpp_driver.o + cp -f libAFLDrivernew.a $LIB_FUZZING_ENGINE +} + +# Provide a way to document the afl++ options used in this build: +echo +echo afl++ target compilation setup: +env | grep AFL_ | tee "$OUT/afl_options.txt" +echo + +popd > /dev/null echo " done." diff --git a/infra/base-images/base-builder/compile_go_fuzzer b/infra/base-images/base-builder/compile_go_fuzzer index 8f8cde759..2342800fb 100755 --- a/infra/base-images/base-builder/compile_go_fuzzer +++ b/infra/base-images/base-builder/compile_go_fuzzer @@ -18,27 +18,35 @@ path=$1 function=$2 fuzzer=$3 -tags="" +tags="-tags gofuzz" if [[ $# -eq 4 ]]; then tags="-tags $4" fi +# makes directory change temporary +( +cd $GOPATH/src/$path || true +# in the case we are in the right directory, with go.mod but no go.sum +go mod tidy || true +# project was downloaded with go get if go list fails +go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; } +# project does not have go.mod if go list fails again +go list $tags $path || { go mod init $path && go mod tidy ;} + if [[ $SANITIZER = *coverage* ]]; then - cd $GOPATH/src/$path - fuzzed_package=`go list $tags -f '{{.Name}}'` + fuzzed_package=`go list $tags -f '{{.Name}}' $path` + abspath=`go list $tags -f {{.Dir}} $path` + cd $abspath cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go - echo "#!/bin/sh" > $OUT/$fuzzer - echo "cd $path" >> $OUT/$fuzzer - # The fuzzer may be in a subdirectory, but we want the coverage report for the whole repository fuzzed_repo=`echo $path | cut -d/ -f-3` - echo "go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -coverprofile \$1 " >> $OUT/$fuzzer - chmod +x $OUT/$fuzzer - - cd - + abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` + # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir + echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath + go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path else # Compile and instrument all Go files relevant to this fuzz target. echo "Running go-fuzz $tags -func $function -o $fuzzer.a $path" @@ -47,3 +55,4 @@ else # Link Go code ($fuzzer.a) with fuzzing engine to produce fuzz target binary. $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer fi +) diff --git a/infra/base-images/base-builder/compile_honggfuzz b/infra/base-images/base-builder/compile_honggfuzz index 362a0a598..f86e8426d 100755 --- a/infra/base-images/base-builder/compile_honggfuzz +++ b/infra/base-images/base-builder/compile_honggfuzz @@ -17,8 +17,8 @@ echo "Skipping compilation; using precompiled honggfuzz" -cp $PRECOMPILED_DIR/honggfuzz.a $LIB_FUZZING_ENGINE -cp $PRECOMPILED_DIR/honggfuzz $OUT/ +cp $SRC/honggfuzz/honggfuzz.a $LIB_FUZZING_ENGINE +cp $SRC/honggfuzz/honggfuzz $OUT/ # Custom coverage flags, roughly in sync with: # https://github.com/google/honggfuzz/blob/oss-fuzz/hfuzz_cc/hfuzz-cc.c diff --git a/infra/base-images/base-builder/compile_libfuzzer b/infra/base-images/base-builder/compile_libfuzzer index 00f2d6337..3fd7f3906 100755 --- a/infra/base-images/base-builder/compile_libfuzzer +++ b/infra/base-images/base-builder/compile_libfuzzer @@ -16,16 +16,7 @@ ################################################################################ echo -n "Compiling libFuzzer to $LIB_FUZZING_ENGINE... " -mkdir -p $WORK/libfuzzer -pushd $WORK/libfuzzer > /dev/null - -# Use -fPIC to allow preloading (LD_PRELOAD). -$CXX $CXXFLAGS -std=c++11 -O2 -fPIC $SANITIZER_FLAGS -fno-sanitize=vptr \ - -c $SRC/libfuzzer/*.cpp -I$SRC/libfuzzer -ar r $LIB_FUZZING_ENGINE_DEPRECATED $WORK/libfuzzer/*.o -popd > /dev/null -rm -rf $WORK/libfuzzer -# Override variable as libFuzzer builds do not link directly against an -# engine library, but use -fsanitize=fuzzer to instruct clang to do so. export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" +cp /usr/local/lib/clang/*/lib/linux/libclang_rt.fuzzer-$ARCHITECTURE.a \ + $LIB_FUZZING_ENGINE_DEPRECATED echo " done." diff --git a/infra/base-images/base-builder/debug_afl b/infra/base-images/base-builder/debug_afl new file mode 100755 index 000000000..c53dae815 --- /dev/null +++ b/infra/base-images/base-builder/debug_afl @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Source this file for afl++ debug sessions. +apt-get update +apt-get install -y strace gdb vim joe psmisc + +pushd $SRC/aflplusplus > /dev/null +git checkout dev +git pull +test -n "$1" && { git checkout "$1" ; git pull ; } +CFLAGS_SAVE="$CFLAGS" +CXXFLAGS_SAVE="$CXXFLAGS" +unset CFLAGS +unset CXXFLAGS +make +export CFLAGS="$CFLAGS_SAVE" +export CXXFLAGS="$CXXFLAGS_SAVE" +popd > /dev/null + +export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1" +export AFL_LLVM_LAF_ALL=1 +export AFL_LLVM_CMPLOG=1 +touch "$OUT/afl_cmplog.txt" +export AFL_LLVM_DICT2FILE=$OUT/afl++.dict +ulimit -c unlimited diff --git a/infra/base-images/base-builder/detect_repo.py b/infra/base-images/base-builder/detect_repo.py index 8969e974f..e677e1023 100644 --- a/infra/base-images/base-builder/detect_repo.py +++ b/infra/base-images/base-builder/detect_repo.py @@ -107,20 +107,25 @@ def get_repo(repo_path): return None -def check_for_repo_name(repo_path, repo_name): - """Check to see if the repo_name matches the remote repository repo name. +def check_for_repo_name(repo_path, expected_repo_name): + """Returns True if the repo at |repo_path| repo_name matches + |expected_repo_name|. Args: - repo_path: The directory of the git repo. - repo_name: The name of the target git repo. + repo_path: The directory of a git repo. + expected_repo_name: The name of the target git repo. """ if not os.path.exists(os.path.join(repo_path, '.git')): return False - out, _ = execute(['git', 'config', '--get', 'remote.origin.url'], - location=repo_path) - out = out.split('/')[-1].replace('.git', '').rstrip() - return out == repo_name + repo_url, _ = execute(['git', 'config', '--get', 'remote.origin.url'], + location=repo_path) + # Handle two common cases: + # https://github.com/google/syzkaller/ + # https://github.com/google/syzkaller.git + repo_url = repo_url.replace('.git', '').rstrip().rstrip('/') + actual_repo_name = repo_url.split('/')[-1] + return actual_repo_name == expected_repo_name def check_for_commit(repo_path, commit): diff --git a/infra/base-images/base-builder/detect_repo_test.py b/infra/base-images/base-builder/detect_repo_test.py index 21f64af44..0243b3ac5 100644 --- a/infra/base-images/base-builder/detect_repo_test.py +++ b/infra/base-images/base-builder/detect_repo_test.py @@ -23,6 +23,7 @@ import re import sys import tempfile import unittest +from unittest import mock import detect_repo @@ -36,6 +37,33 @@ import test_repos # pylint: enable=wrong-import-position +class TestCheckForRepoName(unittest.TestCase): + """Tests for check_for_repo_name.""" + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('detect_repo.execute', + return_value=('https://github.com/google/syzkaller/', None)) + def test_go_get_style_url(self, _, __): + """Tests that check_for_repo_name works on repos that were downloaded using + go get.""" + self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller')) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('detect_repo.execute', + return_value=('https://github.com/google/syzkaller', None)) + def test_missing_git_and_slash_url(self, _, __): + """Tests that check_for_repo_name works on repos who's URLs do not end in + ".git" or "/".""" + self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller')) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('detect_repo.execute', + return_value=('https://github.com/google/syzkaller.git', None)) + def test_normal_style_repo_url(self, _, __): + """Tests that check_for_repo_name works on normally cloned repos.""" + self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller')) + + @unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 'INTEGRATION_TESTS=1 not set') class DetectRepoIntegrationTest(unittest.TestCase): diff --git a/infra/base-images/base-builder/precompile_afl b/infra/base-images/base-builder/precompile_afl new file mode 100755 index 000000000..d6e71f2c3 --- /dev/null +++ b/infra/base-images/base-builder/precompile_afl @@ -0,0 +1,35 @@ +#!/bin/bash -eu +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +echo "Precompiling AFLplusplus" + +pushd $SRC/aflplusplus > /dev/null +make clean +# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it +# down with sanitizers. +SAVE_CXXFLAGS=$CXXFLAGS +SAVE_CFLAGS=$CFLAGS +unset CXXFLAGS +unset CFLAGS +export AFL_IGNORE_UNKNOWN_ENVS=1 +make clean +AFL_NO_X86=1 PYTHON_INCLUDE=/ make +make -C utils/aflpp_driver + +popd > /dev/null + +echo "Done." diff --git a/infra/base-images/base-builder/precompile_honggfuzz b/infra/base-images/base-builder/precompile_honggfuzz index 2565bb83f..df6bb2b75 100755 --- a/infra/base-images/base-builder/precompile_honggfuzz +++ b/infra/base-images/base-builder/precompile_honggfuzz @@ -15,7 +15,7 @@ # ################################################################################ -echo -n "Precompiling honggfuzz to $PRECOMPILED_DIR..." +echo "Precompiling honggfuzz" export BUILD_OSSFUZZ_STATIC=true PACKAGES=( @@ -37,10 +37,9 @@ CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD" make # libhfuzz.a will be added by CC/CXX linker directly during linking, # but it's defined here to satisfy the build infrastructure -ar rcs $PRECOMPILED_DIR/honggfuzz.a libhfuzz/*.o libhfcommon/*.o -cp honggfuzz $PRECOMPILED_DIR/ +ar rcs honggfuzz.a libhfuzz/*.o libhfcommon/*.o popd > /dev/null apt-get remove -y --purge ${PACKAGES[@]} apt-get autoremove -y -echo " done." +echo "Done." diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 928e7934f..3c16a8f3c 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -25,7 +25,8 @@ RUN apt-get update && apt-get install -y wget sudo && \ chmod +x cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ ./cmake-$CMAKE_VERSION-Linux-x86_64.sh --skip-license --prefix="/usr/local" && \ rm cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ - SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo + SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo && \ + rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui COPY checkout_build_install_llvm.sh /root/ # Keep all steps in the same script to decrease the number of intermediate diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh index c5d97d5bf..f6e8ca99c 100755 --- a/infra/base-images/base-clang/checkout_build_install_llvm.sh +++ b/infra/base-images/base-clang/checkout_build_install_llvm.sh @@ -15,10 +15,14 @@ # ################################################################################ -NPROC=16 # See issue #4270. The compiler crashes on GCB instance with 32 vCPUs. +# See issue #4270. The compiler crashes on GCB instance with 32 vCPUs, so when +# we compile on GCB we want 16 cores. But locally we want more (so use nproc / +# 2). +NPROC=$(expr $(nproc) / 2) -LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev" -apt-get install -y $LLVM_DEP_PACKAGES +# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler +LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev" +apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends # Checkout CHECKOUT_RETRIES=10 @@ -60,7 +64,7 @@ function cmake_llvm { # Use chromium's clang revision mkdir $SRC/chromium_tools cd $SRC/chromium_tools -git clone https://chromium.googlesource.com/chromium/src/tools/clang +git clone https://chromium.googlesource.com/chromium/src/tools/clang --depth 1 cd clang LLVM_SRC=$SRC/llvm-project @@ -89,11 +93,9 @@ fi git -C $LLVM_SRC checkout $LLVM_REVISION echo "Using LLVM revision: $LLVM_REVISION" -# Build & install. We build clang in two stages because gcc can't build a -# static version of libcxxabi -# (see https://github.com/google/oss-fuzz/issues/2164). +# Build & install. mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1 -cd $WORK/llvm-stage1 +python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1 TARGET_TO_BUILD= case $(uname -m) in @@ -111,9 +113,6 @@ esac PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld" -cmake_llvm -ninja -j $NPROC - cd $WORK/llvm-stage2 export CC=$WORK/llvm-stage1/bin/clang export CXX=$WORK/llvm-stage1/bin/clang++ @@ -175,3 +174,54 @@ rm -rf $LLVM_SRC rm -rf $SRC/chromium_tools apt-get remove --purge -y $LLVM_DEP_PACKAGES apt-get autoremove -y + +# Delete unneeded parts of LLVM to reduce image size. +# See https://github.com/google/oss-fuzz/issues/5170 +LLVM_TOOLS_TMPDIR=/tmp/llvm-tools +mkdir $LLVM_TOOLS_TMPDIR +# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR +mv \ + /usr/local/bin/llvm-ar \ + /usr/local/bin/llvm-as \ + /usr/local/bin/llvm-config \ + /usr/local/bin/llvm-cov \ + /usr/local/bin/llvm-objcopy \ + /usr/local/bin/llvm-profdata \ + /usr/local/bin/llvm-ranlib \ + /usr/local/bin/llvm-symbolizer \ + /usr/local/bin/llvm-undname \ + $LLVM_TOOLS_TMPDIR +# Delete remaining llvm- binaries. +rm -rf /usr/local/bin/llvm-* +# Restore the llvm- binaries we want to keep. +mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/ +rm -rf $LLVM_TOOLS_TMPDIR + +# Remove binaries from LLVM buld that we don't need. +rm -f \ + /usr/local/bin/bugpoint \ + /usr/local/bin/llc \ + /usr/local/bin/lli \ + /usr/local/bin/clang-check \ + /usr/local/bin/clang-refactor \ + /usr/local/bin/clang-offload-wrapper \ + /usr/local/bin/clang-offload-bundler \ + /usr/local/bin/clang-check \ + /usr/local/bin/clang-refactor \ + /usr/local/bin/c-index-test \ + /usr/local/bin/clang-rename \ + /usr/local/bin/clang-scan-deps \ + /usr/local/bin/clang-extdef-mapping \ + /usr/local/bin/diagtool \ + /usr/local/bin/sanstats \ + /usr/local/bin/dsymutil \ + /usr/local/bin/verify-uselistorder \ + /usr/local/bin/clang-format + +# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the +# libraries. +# Note: we need fuzzer_no_main libraries for atheris. Don't delete. +rm -rf \ + /usr/local/lib/libclang* \ + /usr/local/lib/liblld* \ + /usr/local/lib/cmake/ diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile index f0a264fa7..f847de026 100644..100755 --- a/infra/base-images/base-runner/Dockerfile +++ b/infra/base-images/base-runner/Dockerfile @@ -14,30 +14,40 @@ # ################################################################################ +# Build rust stuff in its own image. We only need the resulting binaries. +# Keeping the rust toolchain in the image wastes 1 GB. +FROM gcr.io/oss-fuzz-base/base-image as temp-runner-binary-builder + +RUN apt-get update && apt-get install -y cargo +RUN cargo install rustfilt + # Using multi-stage build to copy some LLVM binaries needed in the runner image. FROM gcr.io/oss-fuzz-base/base-clang AS base-clang +# Real image that will be used later. FROM gcr.io/oss-fuzz-base/base-image +COPY --from=temp-runner-binary-builder /root/.cargo/bin/rustfilt /usr/local/bin + # Copy the binaries needed for code coverage and crash symbolization. COPY --from=base-clang /usr/local/bin/llvm-cov \ /usr/local/bin/llvm-profdata \ /usr/local/bin/llvm-symbolizer \ /usr/local/bin/ -# TODO(metzman): Install libc6-i386 lib32gcc1 instead of libc6-dev-i386 for -# consistency with ClusterFuzz image and to reduce size. RUN apt-get update && apt-get install -y \ binutils \ file \ fonts-dejavu \ git \ - libc6-dev-i386 \ + lib32gcc1 \ + libc6-i386 \ libcap2 \ python3 \ python3-pip \ + unzip \ wget \ - zip + zip --no-install-recommends RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage /opt/code_coverage && \ pip3 install -r /opt/code_coverage/requirements.txt @@ -66,7 +76,20 @@ ENV GOPATH /root/go ENV PATH $PATH:/root/.go/bin:$GOPATH/bin # Set up Golang coverage modules. -RUN go get github.com/google/oss-fuzz/infra/go/coverage/... +COPY gocoverage $GOPATH/gocoverage +RUN cd $GOPATH/gocoverage && go install ./... + +# Install OpenJDK 15 and trim its size by removing unused components. +ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64 +ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server +ENV PATH=$PATH:$JAVA_HOME/bin + +RUN wget https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz -O /tmp/openjdk-15.0.2_linux-x64_bin.tar.gz && \ + cd /tmp && \ + mkdir -p $JAVA_HOME && \ + tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ + rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \ + rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip # Do this last to make developing these files easier/faster due to caching. COPY bad_build_check \ @@ -75,12 +98,11 @@ COPY bad_build_check \ coverage_helper \ dataflow_tracer.py \ download_corpus \ - minijail0 \ + rcfilt \ reproduce \ run_fuzzer \ - run_minijail \ parse_options.py \ targets_list \ test_all.py \ - test_one \ + test_one.py \ /usr/local/bin/ diff --git a/infra/base-images/base-runner/bad_build_check b/infra/base-images/base-runner/bad_build_check index a57a48252..01f8fbbab 100755 --- a/infra/base-images/base-runner/bad_build_check +++ b/infra/base-images/base-runner/bad_build_check @@ -301,6 +301,12 @@ function check_mixed_sanitizers { local result=0 local CALL_INSN= + if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then + # Sanitizer runtime is linked into the Jazzer driver, so this check does not + # apply. + return 0 + fi + if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then # Sanitizer runtime is loaded via LD_PRELOAD, so this check does not apply. return 0 @@ -312,7 +318,7 @@ function check_mixed_sanitizers { else case $(uname -m) in x86_64) - CALL_INSN="callq\s+[0-9a-f]+\s+<" + CALL_INSN="callq?\s+[0-9a-f]+\s+<" ;; aarch64) CALL_INSN="bl\s+[0-9a-f]+\s+<" @@ -328,6 +334,7 @@ function check_mixed_sanitizers { local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c) local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c) + if [[ "$SANITIZER" = address ]]; then check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS result=$? @@ -340,6 +347,9 @@ function check_mixed_sanitizers { elif [[ "$SANITIZER" = undefined ]]; then check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS result=$? + elif [[ "$SANITIZER" = thread ]]; then + # TODO(metzman): Implement this. + result=0 fi return $result @@ -376,6 +386,12 @@ function check_architecture { local FUZZER=$1 local FUZZER_NAME=$(basename $FUZZER) + if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then + # The native dependencies of a JVM project are not packaged, but loaded + # dynamically at runtime and thus cannot be checked here. + return 0; + fi + if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then FUZZER=${FUZZER}.pkg fi diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 2fcf9e977..a86b00dec 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -114,13 +114,15 @@ function run_fuzz_target { function run_go_fuzz_target { local target=$1 - cd $GOPATH/src echo "Running go target $target" export FUZZ_CORPUS_DIR="/corpus/${target}/" export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf" - bash $OUT/$target $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log + $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log + # translate from golangish paths to current absolute paths + cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done + # cf PATH_EQUIVALENCE_ARGS + sed -i 's=/='$OUT'/=' $DUMPS_DIR/$target.profdata $SYSGOPATH/bin/gocovsum $DUMPS_DIR/$target.profdata > $FUZZER_STATS_DIR/$target.json - cd $OUT } export SYSGOPATH=$GOPATH @@ -131,7 +133,7 @@ for fuzz_target in $FUZZ_TARGETS; do if [[ $FUZZING_LANGUAGE == "go" ]]; then # Continue if not a fuzz target. if [[ $FUZZING_ENGINE != "none" ]]; then - grep "go test -run" $fuzz_target > /dev/null 2>&1 || continue + grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || continue fi run_go_fuzz_target $fuzz_target & else @@ -193,7 +195,7 @@ else # Generate HTML report. llvm-cov show -format=html -output-dir=$REPORT_ROOT_DIR \ - -Xdemangler c++filt -Xdemangler -n $LLVM_COV_ARGS + -Xdemangler rcfilt $LLVM_COV_ARGS # Export coverage summary in JSON format. llvm-cov export -summary-only $LLVM_COV_ARGS > $SUMMARY_FILE diff --git a/infra/base-images/base-runner/gocoverage/go.mod b/infra/base-images/base-runner/gocoverage/go.mod new file mode 100644 index 000000000..b0b57216e --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/go.mod @@ -0,0 +1,8 @@ +module oss-fuzz.com/gocoverage + +go 1.14 + +require ( + github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 + golang.org/x/tools v0.1.0 +) diff --git a/infra/base-images/base-runner/gocoverage/go.sum b/infra/base-images/base-runner/gocoverage/go.sum new file mode 100644 index 000000000..3279af3ba --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/go.sum @@ -0,0 +1,30 @@ +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/infra/go/coverage/gocovmerge/LICENSE b/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE index 455fb1087..455fb1087 100644 --- a/infra/go/coverage/gocovmerge/LICENSE +++ b/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE diff --git a/infra/go/coverage/gocovmerge/gocovmerge.go b/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go index e8099839e..e8099839e 100644 --- a/infra/go/coverage/gocovmerge/gocovmerge.go +++ b/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go diff --git a/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go b/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go new file mode 100644 index 000000000..973b7ae92 --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go @@ -0,0 +1,147 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + + "go/ast" + "go/parser" + "go/token" + + "golang.org/x/tools/cover" +) + +type CoverageTotal struct { + Count int `json:"count"` + Covered int `json:"covered"` + Uncovered int `json:"notcovered"` + Percent float64 `json:"percent"` +} + +type CoverageTotals struct { + Functions CoverageTotal `json:"functions,omitempty"` + Lines CoverageTotal `json:"lines,omitempty"` + Regions CoverageTotal `json:"regions,omitempty"` + Instantiations CoverageTotal `json:"instantiations,omitempty"` + Branches CoverageTotal `json:"branches,omitempty"` +} + +type CoverageFile struct { + Summary CoverageTotals `json:"summary,omitempty"` + Filename string `json:"filename,omitempty"` +} + +type CoverageData struct { + Totals CoverageTotals `json:"totals,omitempty"` + Files []CoverageFile `json:"files,omitempty"` +} + +type PositionInterval struct { + start token.Position + end token.Position +} + +type CoverageSummary struct { + Data []CoverageData `json:"data,omitempty"` + Type string `json:"type,omitempty"` + Version string `json:"version,omitempty"` +} + +func isFunctionCovered(s token.Position, e token.Position, blocks []cover.ProfileBlock) bool { + for _, b := range blocks { + if b.StartLine >= s.Line && b.StartLine <= e.Line && b.EndLine >= s.Line && b.EndLine <= e.Line { + if b.Count > 0 { + return true + } + } + } + return false +} + +func computePercent(s *CoverageTotals) { + s.Regions.Percent = float64(100*s.Regions.Covered) / float64(s.Regions.Count) + s.Lines.Percent = float64(100*s.Lines.Covered) / float64(s.Lines.Count) + s.Functions.Percent = float64(100*s.Functions.Covered) / float64(s.Functions.Count) +} + +func main() { + flag.Parse() + + if len(flag.Args()) != 1 { + log.Fatalf("needs exactly one argument") + } + profiles, err := cover.ParseProfiles(flag.Args()[0]) + if err != nil { + log.Fatalf("failed to parse profiles: %v", err) + } + r := CoverageSummary{} + r.Type = "oss-fuzz.go.coverage.json.export" + r.Version = "2.0.1" + r.Data = make([]CoverageData, 1) + for _, p := range profiles { + fset := token.NewFileSet() // positions are relative to fset + f, err := parser.ParseFile(fset, p.FileName, nil, 0) + if err != nil { + panic(err) + } + fileCov := CoverageFile{} + fileCov.Filename = p.FileName + ast.Inspect(f, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.FuncLit: + startf := fset.Position(x.Pos()) + endf := fset.Position(x.End()) + fileCov.Summary.Functions.Count++ + if isFunctionCovered(startf, endf, p.Blocks) { + fileCov.Summary.Functions.Covered++ + } else { + fileCov.Summary.Functions.Uncovered++ + } + case *ast.FuncDecl: + startf := fset.Position(x.Pos()) + endf := fset.Position(x.End()) + fileCov.Summary.Functions.Count++ + if isFunctionCovered(startf, endf, p.Blocks) { + fileCov.Summary.Functions.Covered++ + } else { + fileCov.Summary.Functions.Uncovered++ + } + } + return true + }) + + for _, b := range p.Blocks { + fileCov.Summary.Regions.Count++ + if b.Count > 0 { + fileCov.Summary.Regions.Covered++ + } else { + fileCov.Summary.Regions.Uncovered++ + } + + fileCov.Summary.Lines.Count += b.NumStmt + if b.Count > 0 { + fileCov.Summary.Lines.Covered += b.NumStmt + } else { + fileCov.Summary.Lines.Uncovered += b.NumStmt + } + } + r.Data[0].Totals.Regions.Count += fileCov.Summary.Regions.Count + r.Data[0].Totals.Regions.Covered += fileCov.Summary.Regions.Covered + r.Data[0].Totals.Regions.Uncovered += fileCov.Summary.Regions.Uncovered + r.Data[0].Totals.Lines.Count += fileCov.Summary.Lines.Count + r.Data[0].Totals.Lines.Covered += fileCov.Summary.Lines.Covered + r.Data[0].Totals.Lines.Uncovered += fileCov.Summary.Lines.Uncovered + r.Data[0].Totals.Functions.Count += fileCov.Summary.Functions.Count + r.Data[0].Totals.Functions.Covered += fileCov.Summary.Functions.Covered + r.Data[0].Totals.Functions.Uncovered += fileCov.Summary.Functions.Uncovered + + computePercent(&fileCov.Summary) + r.Data[0].Files = append(r.Data[0].Files, fileCov) + } + + computePercent(&r.Data[0].Totals) + o, _ := json.Marshal(r) + fmt.Printf(string(o)) +} diff --git a/infra/go/coverage/pprof-merge/LICENSE b/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE index 8dada3eda..8dada3eda 100644 --- a/infra/go/coverage/pprof-merge/LICENSE +++ b/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE diff --git a/infra/go/coverage/pprof-merge/main.go b/infra/base-images/base-runner/gocoverage/pprof-merge/main.go index f35156403..f35156403 100644 --- a/infra/go/coverage/pprof-merge/main.go +++ b/infra/base-images/base-runner/gocoverage/pprof-merge/main.go diff --git a/infra/base-images/base-runner/minijail0 b/infra/base-images/base-runner/minijail0 Binary files differdeleted file mode 100755 index 369e0bbd9..000000000 --- a/infra/base-images/base-runner/minijail0 +++ /dev/null diff --git a/infra/base-images/base-runner/rcfilt b/infra/base-images/base-runner/rcfilt new file mode 100755 index 000000000..1c621100c --- /dev/null +++ b/infra/base-images/base-runner/rcfilt @@ -0,0 +1,21 @@ +#!/bin/bash -u +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Symbol demangling for both C++ and Rust +# +################################################################################ + +# simply pipe +rustfilt | c++filt -n diff --git a/infra/base-images/base-runner/run_fuzzer b/infra/base-images/base-runner/run_fuzzer index 6464ddc2c..b9bc8d9d6 100755 --- a/infra/base-images/base-runner/run_fuzzer +++ b/infra/base-images/base-runner/run_fuzzer @@ -98,22 +98,29 @@ fi if [[ "$FUZZING_ENGINE" = afl ]]; then # Set afl++ environment options. - export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1:symbolize=0" + export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1:symbolize=0:detect_odr_violation=0:" export MSAN_OPTIONS="$MSAN_OPTIONS:exit_code=86:symbolize=0" export UBSAN_OPTIONS="$UBSAN_OPTIONS:symbolize=0" export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 export AFL_SKIP_CPUFREQ=1 export AFL_NO_AFFINITY=1 export AFL_FAST_CAL=1 - export AFL_MAP_SIZE=4194304 # If $OUT/afl_cmplog.txt is present this means the target was compiled for # CMPLOG. So we have to add the proper parameters to afl-fuzz. `-l 2` is # CMPLOG level 2, which will colorize larger files but not huge files and # not enable transform analysis unless there have been several cycles without # any finds. - test -e $OUT/afl_cmplog.txt && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -l 2 -c $OUT/$FUZZER" + test -e "$OUT/afl_cmplog.txt" && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -l 2 -c $OUT/$FUZZER" + # If $OUT/afl++.dict we load it as a dictionary for afl-fuzz. + test -e "$OUT/afl++.dict" && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -x $OUT/afl++.dict" + # Ensure timeout is a bit large than 1sec as some of the OSS-Fuzz fuzzers + # are slower than this. + AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -t 5000+" # AFL expects at least 1 file in the input dir. echo input > ${CORPUS_DIR}/input + echo afl++ setup: + env|grep AFL_ + cat "$OUT/afl_options.txt" CMD_LINE="$OUT/afl-fuzz $AFL_FUZZER_ARGS -i $CORPUS_DIR -o $FUZZER_OUT $(get_dictionary) $* -- $OUT/$FUZZER" elif [[ "$FUZZING_ENGINE" = honggfuzz ]]; then diff --git a/infra/base-images/base-runner/run_minijail b/infra/base-images/base-runner/run_minijail deleted file mode 100755 index bf950b1e7..000000000 --- a/infra/base-images/base-runner/run_minijail +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -eu -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -rm -rf /tmp/chroot -mkdir /tmp/chroot - -mkdir /tmp/chroot/lib -mkdir /tmp/chroot/lib64 -mkdir /tmp/chroot/lib32 - -mkdir /tmp/chroot/usr -mkdir /tmp/chroot/usr/lib -mkdir /tmp/chroot/usr/lib32 - -mkdir /tmp/chroot/dev -mknod -m 666 /tmp/chroot/dev/null c 1 3 -mknod -m 666 /tmp/chroot/dev/urandom c 1 9 - -mkdir /tmp/chroot/proc -mkdir /tmp/chroot/tmp - -mkdir /tmp/chroot/bin -cp /bin/sh /tmp/chroot/bin/sh -cp $(which llvm-symbolizer) /tmp/chroot/bin/llvm-symbolizer - -FULL_EXE_PATH=$(readlink -f $1) -EXE_DIR=$(dirname $FULL_EXE_PATH) -mkdir -p /tmp/chroot/$EXE_DIR - -shift - -echo 'Running:' -echo minijail0 -U -m \"0 $UID 1\" -T static \ - -c 0 -n -v -p -l -I \ - -k proc,/proc,proc,1 -P /tmp/chroot \ - -b /lib,/lib,0 -b /lib64,/lib64,0 -b /lib32,/lib32,0 -b /usr/lib,/usr/lib,0 \ - -b /usr/lib32,/usr/lib32,0 -b /tmp,/tmp,1 \ - -b $EXE_DIR,$EXE_DIR,0 $FULL_EXE_PATH $@ - -minijail0 -U -m "0 $UID 1" -T static \ - -c 0 -n -v -p -l -I \ - -k proc,/proc,proc,1 -P /tmp/chroot \ - -b /lib,/lib,0 -b /lib64,/lib64,0 -b /lib32,/lib32,0 -b /usr/lib,/usr/lib,0 \ - -b /usr/lib32,/usr/lib32,0 -b /tmp,/tmp,1 \ - -b $EXE_DIR,$EXE_DIR,0 $FULL_EXE_PATH $@ diff --git a/infra/base-images/base-runner/test_all.py b/infra/base-images/base-runner/test_all.py index 360da0345..925ebde69 100755 --- a/infra/base-images/base-runner/test_all.py +++ b/infra/base-images/base-runner/test_all.py @@ -78,11 +78,16 @@ def find_fuzz_targets(directory, fuzzing_language): continue if filename.startswith('afl-'): continue + if filename.startswith('jazzer_'): + continue if not os.path.isfile(path): continue if not os.stat(path).st_mode & EXECUTABLE: continue - if fuzzing_language != 'python' and not is_elf(path): + # Fuzz targets are expected to be ELF binaries for languages other than + # Python and Java. + if (fuzzing_language != 'python' and fuzzing_language != 'jvm' and + not is_elf(path)): continue if os.getenv('FUZZING_ENGINE') != 'none': with open(path, 'rb') as file_handle: diff --git a/infra/base-images/base-runner/test_one b/infra/base-images/base-runner/test_one deleted file mode 100755 index 23b7fd932..000000000 --- a/infra/base-images/base-runner/test_one +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -u -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# Wrapper around bad_build_check that moves the /out directory to /tmp/not-out. -# This is useful when bad_build_check isn't called from test_all which does the -# same thing. - -function main { - # Move the directory the fuzzer is located in to somewhere that doesn't exist - # on the builder to make it more likely that hardcoding /out fails here (since - # it will fail on ClusterFuzz). - local fuzzer=$1 - fuzzer=$(realpath $fuzzer) - local initial_fuzzer_dir=$(dirname $fuzzer) - - local tmp_fuzzer_dir=/tmp/not-out - rm -rf $tmp_fuzzer_dir - mkdir $tmp_fuzzer_dir - # Move the contents of $initial_fuzzer_dir rather than the directory itself in - # case it is a mount. - mv $initial_fuzzer_dir/* $tmp_fuzzer_dir - fuzzer="$tmp_fuzzer_dir/$(basename $fuzzer)" - - # Change OUT to the temporary fuzzer dir. - local initial_out=$OUT - export OUT=$tmp_fuzzer_dir - - bad_build_check $fuzzer - returncode=$? - - # Restore OUT and $initial_fuzzer_dir - export OUT=$initial_out - mv $tmp_fuzzer_dir/* $initial_fuzzer_dir - - return $returncode -} - -if [ $# -ne 1 ]; then - echo "Usage: $0 <fuzz_target_binary>" - exit 1 -fi - -main $1 -exit $? diff --git a/infra/base-images/base-runner/test_one.py b/infra/base-images/base-runner/test_one.py new file mode 100755 index 000000000..9bdb75faf --- /dev/null +++ b/infra/base-images/base-runner/test_one.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Does bad_build_check on a fuzz target in $OUT.""" +import os +import sys + +import test_all + + +def test_one(fuzz_target): + """Does bad_build_check on one fuzz target. Returns True on success.""" + with test_all.use_different_out_dir(): + fuzz_target_path = os.path.join(os.environ['OUT'], fuzz_target) + return test_all.do_bad_build_check(fuzz_target_path).returncode == 0 + + +def main(): + """Does bad_build_check on one fuzz target. Returns 1 on failure, 0 on + success.""" + if len(sys.argv) != 2: + print('Usage: %d <fuzz_target>', sys.argv[0]) + return 1 + + fuzz_target_binary = sys.argv[1] + return 0 if test_one(fuzz_target_binary) else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/base-images/base-sanitizer-libs-builder/msan_build.py b/infra/base-images/base-sanitizer-libs-builder/msan_build.py index 928b1a596..5ea00ab10 100755 --- a/infra/base-images/base-sanitizer-libs-builder/msan_build.py +++ b/infra/base-images/base-sanitizer-libs-builder/msan_build.py @@ -73,7 +73,9 @@ def SetUpEnvironment(work_dir): dpkg_host_architecture = wrapper_utils.DpkgHostArchitecture() wrapper_utils.CreateSymlinks( - compiler_wrapper_path, bin_dir, [ + compiler_wrapper_path, + bin_dir, + [ 'clang', 'clang++', # Not all build rules respect $CC/$CXX, so make additional symlinks. @@ -101,41 +103,35 @@ def SetUpEnvironment(work_dir): env['DPKG_GENSYMBOLS_CHECK_LEVEL'] = '0' # debian/rules can set DPKG_GENSYMBOLS_CHECK_LEVEL explicitly, so override it. - gen_symbols_wrapper = ( - '#!/bin/sh\n' - 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n' - '/usr/bin/dpkg-gensymbols "$@"\n') + gen_symbols_wrapper = ('#!/bin/sh\n' + 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n' + '/usr/bin/dpkg-gensymbols "$@"\n') - wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols', - gen_symbols_wrapper) + wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols', gen_symbols_wrapper) # Install no-op strip binaries. - no_op_strip = ('#!/bin/sh\n' - 'exit 0\n') - wrapper_utils.InstallWrapper( - bin_dir, 'strip', no_op_strip, - [dpkg_host_architecture + '-strip']) + no_op_strip = ('#!/bin/sh\n' 'exit 0\n') + wrapper_utils.InstallWrapper(bin_dir, 'strip', no_op_strip, + [dpkg_host_architecture + '-strip']) env['PATH'] = bin_dir + ':' + os.environ['PATH'] # nocheck doesn't disable override_dh_auto_test. So we have this hack to try # to disable "make check" or "make test" invocations. - make_wrapper = ( - '#!/bin/bash\n' - 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n' - ' exit 0\n' - 'fi\n' - '/usr/bin/make "$@"\n') - wrapper_utils.InstallWrapper(bin_dir, 'make', - make_wrapper) + make_wrapper = ('#!/bin/bash\n' + 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n' + ' exit 0\n' + 'fi\n' + '/usr/bin/make "$@"\n') + wrapper_utils.InstallWrapper(bin_dir, 'make', make_wrapper) # Prevent entire build from failing because of bugs/uninstrumented in tools # that are part of the build. msan_log_dir = os.path.join(work_dir, 'msan') os.mkdir(msan_log_dir) msan_log_path = os.path.join(msan_log_dir, 'log') - env['MSAN_OPTIONS'] = ( - 'halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + msan_log_path) + env['MSAN_OPTIONS'] = ('halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + + msan_log_path) # Increase maximum stack size to prevent tests from failing. limit = 128 * 1024 * 1024 @@ -207,7 +203,7 @@ def ExtractLibraries(deb_paths, work_directory, output_directory): target_file_path = os.path.join(output_directory, rel_file_path) extracted.append(target_file_path) - + if os.path.lexists(target_file_path): os.remove(target_file_path) @@ -215,8 +211,8 @@ def ExtractLibraries(deb_paths, work_directory, output_directory): link_path = os.readlink(file_path) if os.path.isabs(link_path): # Make absolute links relative. - link_path = os.path.relpath( - link_path, os.path.join('/', rel_directory)) + link_path = os.path.relpath(link_path, + os.path.join('/', rel_directory)) os.symlink(link_path, target_file_path) else: @@ -244,8 +240,8 @@ def GetPackage(package_name): def PatchRpath(path, output_directory): """Patch rpath to be relative to $ORIGIN.""" try: - rpaths = subprocess.check_output( - ['patchelf', '--print-rpath', path]).strip() + rpaths = subprocess.check_output(['patchelf', '--print-rpath', + path]).strip() except subprocess.CalledProcessError: return @@ -262,15 +258,13 @@ def PatchRpath(path, output_directory): processed_rpath.append(rpath) continue - processed_rpath.append(os.path.join( - '$ORIGIN', - os.path.relpath(rpath, rel_directory))) + processed_rpath.append( + os.path.join('$ORIGIN', os.path.relpath(rpath, rel_directory))) processed_rpath = ':'.join(processed_rpath) print('Patching rpath for', path, 'to', processed_rpath) subprocess.check_call( - ['patchelf', '--force-rpath', '--set-rpath', - processed_rpath, path]) + ['patchelf', '--force-rpath', '--set-rpath', processed_rpath, path]) def _CollectDependencies(apt_cache, pkg, cache, dependencies): @@ -331,7 +325,11 @@ def GetBuildList(package_name): class MSanBuilder(object): """MSan builder.""" - def __init__(self, debug=False, log_path=None, work_dir=None, no_track_origins=False): + def __init__(self, + debug=False, + log_path=None, + work_dir=None, + no_track_origins=False): self.debug = debug self.log_path = log_path self.work_dir = work_dir @@ -396,19 +394,24 @@ class MSanBuilder(object): extracted_paths = ExtractLibraries(deb_paths, self.work_dir, extract_directory) for extracted_path in extracted_paths: - if not os.path.islink(extracted_path): - PatchRpath(extracted_path, extract_directory) + if os.path.islink(extracted_path): + continue + if os.path.basename(extracted_path) == 'llvm-symbolizer': + continue + PatchRpath(extracted_path, extract_directory) def main(): parser = argparse.ArgumentParser('msan_build.py', description='MSan builder.') parser.add_argument('package_names', nargs='+', help='Name of the packages.') parser.add_argument('output_dir', help='Output directory.') - parser.add_argument('--create-subdirs', action='store_true', + parser.add_argument('--create-subdirs', + action='store_true', help=('Create subdirectories in the output ' 'directory for each package.')) parser.add_argument('--work-dir', help='Work directory.') - parser.add_argument('--no-build-deps', action='store_true', + parser.add_argument('--no-build-deps', + action='store_true', help='Don\'t build dependencies.') parser.add_argument('--debug', action='store_true', help='Enable debug mode.') parser.add_argument('--log-path', help='Log path for debugging.') @@ -445,7 +448,8 @@ def main(): for package_name in package_names: print('\t', package_name) - with MSanBuilder(debug=args.debug, log_path=args.log_path, + with MSanBuilder(debug=args.debug, + log_path=args.log_path, work_dir=args.work_dir, no_track_origins=args.no_track_origins) as builder: for package_name in package_names: diff --git a/infra/bisector.py b/infra/bisector.py index dc4a470d5..1438d0de9 100644 --- a/infra/bisector.py +++ b/infra/bisector.py @@ -189,6 +189,8 @@ def _bisect(bisect_type, old_commit, new_commit, test_case_path, fuzz_target, bisect_repo_manager = repo_manager.RepoManager( os.path.join(host_src_dir, os.path.basename(repo_path))) + bisect_repo_manager.fetch_all_remotes() + commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit) old_idx = len(commit_list) - 1 diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py index 71d7338f9..cc2de5a32 100644 --- a/infra/build/functions/build_and_run_coverage.py +++ b/infra/build/functions/build_and_run_coverage.py @@ -48,7 +48,7 @@ LATEST_REPORT_INFO_CONTENT_TYPE = 'application/json' UPLOAD_URL_FORMAT = 'gs://' + COVERAGE_BUCKET_NAME + '/{project}/{type}/{date}' # Languages from project.yaml that have code coverage support. -LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go'] +LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust'] def usage(): diff --git a/infra/build/functions/requirements.txt b/infra/build/functions/requirements.txt index 8d07d82ff..a60a03283 100644 --- a/infra/build/functions/requirements.txt +++ b/infra/build/functions/requirements.txt @@ -16,7 +16,7 @@ Brotli==1.0.9 hiredis==1.1.0 -PyYaml==5.2 +PyYaml==5.4 PyGithub==1.51 grpcio==1.29.0 google-auth==1.21.1 diff --git a/infra/build/functions/update_build_status.py b/infra/build/functions/update_build_status.py index b5f955227..af65a41ab 100644 --- a/infra/build/functions/update_build_status.py +++ b/infra/build/functions/update_build_status.py @@ -195,7 +195,10 @@ def update_build_badges(project, last_build_successful, last_coverage_build_successful): """Upload badges of given project.""" badge = 'building' - if not last_coverage_build_successful: + # last_coverage_build_successful is False if there was an unsuccessful build + # and None if the target does not support coverage (e.g. Python or Java + # targets). + if last_coverage_build_successful is False: badge = 'coverage_failing' if not last_build_successful: badge = 'failing' @@ -289,12 +292,16 @@ def update_badges(): futures = [] with ndb.Client().context(): for project in Project.query(): - if (project.name not in project_build_statuses or - project.name not in coverage_build_statuses): + if project.name not in project_build_statuses: continue + # Certain projects (e.g. JVM and Python) do not have any coverage + # builds, but should still receive a badge. + coverage_build_status = None + if project.name in coverage_build_statuses: + coverage_build_status = coverage_build_statuses[project.name] futures.append( executor.submit(update_build_badges, project.name, project_build_statuses[project.name], - coverage_build_statuses[project.name])) + coverage_build_status)) concurrent.futures.wait(futures) diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py index 9f29e420e..b2130ea85 100644 --- a/infra/build_specified_commit.py +++ b/infra/build_specified_commit.py @@ -200,6 +200,7 @@ def build_fuzzers_from_commit(commit, # Re-copy /src for a clean checkout every time. copy_src_from_docker(build_data.project_name, os.path.dirname(host_src_path)) + build_repo_manager.fetch_all_remotes() projects_dir = os.path.join('projects', build_data.project_name) dockerfile_path = os.path.join(projects_dir, 'Dockerfile') diff --git a/infra/build_specified_commit_test.py b/infra/build_specified_commit_test.py index 916b31885..a86504580 100644 --- a/infra/build_specified_commit_test.py +++ b/infra/build_specified_commit_test.py @@ -95,6 +95,10 @@ class BuildImageIntegrationTest(unittest.TestCase): def test_detect_main_repo_from_name(self): """Test the detect main repo function from build specific commit module.""" for example_repo in test_repos.TEST_REPOS: + if example_repo.project_name == 'gonids': + # It's unclear how this test ever passed, but we can't infer the repo + # because gonids doesn't really check it out, it uses "go get". + continue repo_origin, repo_name = build_specified_commit.detect_main_repo( example_repo.project_name, repo_name=example_repo.git_repo_name) self.assertEqual(repo_origin, example_repo.git_url) diff --git a/infra/ci/build.py b/infra/ci/build.py index f71799bb2..addeb7879 100755 --- a/infra/ci/build.py +++ b/infra/ci/build.py @@ -32,7 +32,7 @@ DEFAULT_ENGINES = ['afl', 'honggfuzz', 'libfuzzer'] DEFAULT_SANITIZERS = ['address', 'undefined'] # Languages from project.yaml that have code coverage support. -LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go'] +LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust'] def get_changed_files_output(): diff --git a/infra/ci/requirements.txt b/infra/ci/requirements.txt index 48d2ae758..f0a8be0b5 100644 --- a/infra/ci/requirements.txt +++ b/infra/ci/requirements.txt @@ -4,5 +4,5 @@ pyfakefs==4.1.0 pylint==2.5.3 pytest==6.2.1 pytest-xdist==2.2.0 -PyYAML==5.3.1 +PyYAML==5.4 yapf==0.30.0 diff --git a/infra/cifuzz/actions/build_fuzzers/action.yml b/infra/cifuzz/actions/build_fuzzers/action.yml index 2919db40e..835b7b430 100644 --- a/infra/cifuzz/actions/build_fuzzers/action.yml +++ b/infra/cifuzz/actions/build_fuzzers/action.yml @@ -5,6 +5,10 @@ inputs: oss-fuzz-project-name: description: 'Name of the corresponding OSS-Fuzz project.' required: true + language: + description: 'Programming language project is written in.' + required: false + default: 'c++' dry-run: description: 'If set, run the action without actually reporting a failure.' default: false @@ -20,13 +24,20 @@ inputs: build-integration-path: description: "The path to the the project's build integration." required: false + bad-build-check: + description: "Whether or not OSS-Fuzz's check for bad builds should be done." + required: false + default: true runs: using: 'docker' image: '../../../build_fuzzers.Dockerfile' env: OSS_FUZZ_PROJECT_NAME: ${{ inputs.oss-fuzz-project-name }} + LANGUAGE: ${{ inputs.language }} DRY_RUN: ${{ inputs.dry-run}} ALLOWED_BROKEN_TARGETS_PERCENTAGE: ${{ inputs.allowed-broken-targets-percentage}} SANITIZER: ${{ inputs.sanitizer }} PROJECT_SRC_PATH: ${{ inputs.project-src-path }} BUILD_INTEGRATION_PATH: ${{ inputs.build-integration-path }} + LOW_DISK_SPACE: 'True' + BAD_BUILD_CHECK: ${{ inputs.bad-build-check }} diff --git a/infra/cifuzz/actions/run_fuzzers/action.yml b/infra/cifuzz/actions/run_fuzzers/action.yml index 582133c74..d1c03c833 100644 --- a/infra/cifuzz/actions/run_fuzzers/action.yml +++ b/infra/cifuzz/actions/run_fuzzers/action.yml @@ -5,6 +5,10 @@ inputs: oss-fuzz-project-name: description: 'The OSS-Fuzz project name.' required: true + language: + description: 'Programming language project is written in.' + required: false + default: 'c++' fuzz-seconds: description: 'The total time allotted for fuzzing in seconds.' required: true @@ -31,6 +35,7 @@ runs: image: '../../../run_fuzzers.Dockerfile' env: OSS_FUZZ_PROJECT_NAME: ${{ inputs.oss-fuzz-project-name }} + LANGUAGE: ${{ inputs.language }} FUZZ_SECONDS: ${{ inputs.fuzz-seconds }} DRY_RUN: ${{ inputs.dry-run}} SANITIZER: ${{ inputs.sanitizer }} @@ -39,3 +44,4 @@ runs: # for running because we use it to distinguish OSS-Fuzz from non-OSS-Fuzz. # We should do something explicit instead. BUILD_INTEGRATION_PATH: ${{ inputs.build-integration-path }} + LOW_DISK_SPACE: 'True' diff --git a/infra/cifuzz/affected_fuzz_targets_test.py b/infra/cifuzz/affected_fuzz_targets_test.py index 72e6d266c..05f27c072 100644 --- a/infra/cifuzz/affected_fuzz_targets_test.py +++ b/infra/cifuzz/affected_fuzz_targets_test.py @@ -30,15 +30,15 @@ EXAMPLE_PROJECT = 'example' EXAMPLE_FILE_CHANGED = 'test.txt' -TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files') +TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_data') class RemoveUnaffectedFuzzTargets(unittest.TestCase): """Tests remove_unaffected_fuzzers.""" - TEST_FUZZER_1 = os.path.join(TEST_FILES_PATH, 'out', 'example_crash_fuzzer') - TEST_FUZZER_2 = os.path.join(TEST_FILES_PATH, 'out', 'example_nocrash_fuzzer') + TEST_FUZZER_1 = os.path.join(TEST_DATA_PATH, 'out', 'example_crash_fuzzer') + TEST_FUZZER_2 = os.path.join(TEST_DATA_PATH, 'out', 'example_nocrash_fuzzer') # yapf: disable @parameterized.parameterized.expand([ diff --git a/infra/cifuzz/build_fuzzers.py b/infra/cifuzz/build_fuzzers.py index a4342a413..78180b52b 100644 --- a/infra/cifuzz/build_fuzzers.py +++ b/infra/cifuzz/build_fuzzers.py @@ -20,6 +20,7 @@ import sys import affected_fuzz_targets import continuous_integration +import docker # pylint: disable=wrong-import-position,import-error sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -77,7 +78,8 @@ class Builder: # pylint: disable=too-many-instance-attributes def build_fuzzers(self): """Moves the source code we want to fuzz into the project builder and builds the fuzzers from that source code. Returns True on success.""" - docker_args = get_common_docker_args(self.config.sanitizer) + docker_args = get_common_docker_args(self.config.sanitizer, + self.config.language) container = utils.get_container_name() if container: @@ -93,7 +95,7 @@ class Builder: # pylint: disable=too-many-instance-attributes self.handle_msan_prebuild(container) docker_args.extend([ - 'gcr.io/oss-fuzz/' + self.config.project_name, + docker.get_project_image_name(self.config.project_name), '/bin/bash', '-c', ]) @@ -118,8 +120,7 @@ class Builder: # pylint: disable=too-many-instance-attributes helper.docker_run([ '--volumes-from', container, '-e', 'WORK={work_dir}'.format(work_dir=self.work_dir), - 'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 'patch_build.py', - '/out' + docker.MSAN_LIBS_BUILDER_TAG, 'patch_build.py', '/out' ]) def handle_msan_prebuild(self, container): @@ -127,8 +128,8 @@ class Builder: # pylint: disable=too-many-instance-attributes returns docker arguments to use that directory for MSAN libs.""" logging.info('Copying MSAN libs.') helper.docker_run([ - '--volumes-from', container, 'gcr.io/oss-fuzz-base/msan-libs-builder', - 'bash', '-c', 'cp -r /msan {work_dir}'.format(work_dir=self.work_dir) + '--volumes-from', container, docker.MSAN_LIBS_BUILDER_TAG, 'bash', '-c', + 'cp -r /msan {work_dir}'.format(work_dir=self.work_dir) ]) def build(self): @@ -185,7 +186,7 @@ def build_fuzzers(config): return builder.build() -def get_common_docker_args(sanitizer): +def get_common_docker_args(sanitizer, language): """Returns a list of common docker arguments.""" return [ '--cap-add', @@ -199,12 +200,13 @@ def get_common_docker_args(sanitizer): '-e', 'CIFUZZ=True', '-e', - 'FUZZING_LANGUAGE=c++', # FIXME: Add proper support. + 'FUZZING_LANGUAGE=' + language, ] def check_fuzzer_build(out_dir, - sanitizer='address', + sanitizer, + language, allowed_broken_targets_percentage=None): """Checks the integrity of the built fuzzers. @@ -222,7 +224,7 @@ def check_fuzzer_build(out_dir, logging.error('No fuzzers found in out directory: %s.', out_dir) return False - command = get_common_docker_args(sanitizer) + command = get_common_docker_args(sanitizer, language) if allowed_broken_targets_percentage is not None: command += [ @@ -236,7 +238,7 @@ def check_fuzzer_build(out_dir, command += ['-e', 'OUT=' + out_dir, '--volumes-from', container] else: command += ['-v', '%s:/out' % out_dir] - command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all.py']) + command.extend(['-t', docker.BASE_RUNNER_TAG, 'test_all.py']) exit_code = helper.docker_run(command) logging.info('check fuzzer build exit code: %d', exit_code) if exit_code: diff --git a/infra/cifuzz/build_fuzzers_entrypoint.py b/infra/cifuzz/build_fuzzers_entrypoint.py index 9c4b98215..04f562068 100644 --- a/infra/cifuzz/build_fuzzers_entrypoint.py +++ b/infra/cifuzz/build_fuzzers_entrypoint.py @@ -72,10 +72,16 @@ def main(): return returncode out_dir = os.path.join(config.workspace, 'out') + + if not config.bad_build_check: + # If we've gotten to this point and we don't need to do bad_build_check, + # then the build has succeeded. + returncode = 0 # yapf: disable - if build_fuzzers.check_fuzzer_build( + elif build_fuzzers.check_fuzzer_build( out_dir, - sanitizer=config.sanitizer, + config.sanitizer, + config.language, allowed_broken_targets_percentage=config.allowed_broken_targets_percentage ): # yapf: enable diff --git a/infra/cifuzz/build_fuzzers_test.py b/infra/cifuzz/build_fuzzers_test.py index 2d27356d2..298778867 100644 --- a/infra/cifuzz/build_fuzzers_test.py +++ b/infra/cifuzz/build_fuzzers_test.py @@ -36,9 +36,9 @@ import test_helpers # https://github.com/google/oss-fuzz/tree/master/projects/example project. EXAMPLE_PROJECT = 'example' -# Location of files used for testing. -TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files') +# Location of data used for testing. +TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_data') # An example fuzzer that triggers an crash. # Binary is a copy of the example project's do_stuff_fuzzer and can be @@ -251,10 +251,13 @@ class BuildFuzzersIntegrationTest(unittest.TestCase): class CheckFuzzerBuildTest(unittest.TestCase): """Tests the check_fuzzer_build function in the cifuzz module.""" + SANITIZER = 'address' + LANGUAGE = 'c++' + def setUp(self): self.tmp_dir_obj = tempfile.TemporaryDirectory() self.test_files_path = os.path.join(self.tmp_dir_obj.name, 'test_files') - shutil.copytree(TEST_FILES_PATH, self.test_files_path) + shutil.copytree(TEST_DATA_PATH, self.test_files_path) def tearDown(self): self.tmp_dir_obj.cleanup() @@ -262,23 +265,31 @@ class CheckFuzzerBuildTest(unittest.TestCase): def test_correct_fuzzer_build(self): """Checks check_fuzzer_build function returns True for valid fuzzers.""" test_fuzzer_dir = os.path.join(self.test_files_path, 'out') - self.assertTrue(build_fuzzers.check_fuzzer_build(test_fuzzer_dir)) + self.assertTrue( + build_fuzzers.check_fuzzer_build(test_fuzzer_dir, self.SANITIZER, + self.LANGUAGE)) def test_not_a_valid_fuzz_path(self): """Tests that False is returned when a bad path is given.""" - self.assertFalse(build_fuzzers.check_fuzzer_build('not/a/valid/path')) + self.assertFalse( + build_fuzzers.check_fuzzer_build('not/a/valid/path', self.SANITIZER, + self.LANGUAGE)) def test_not_a_valid_fuzzer(self): """Checks a directory that exists but does not have fuzzers is False.""" - self.assertFalse(build_fuzzers.check_fuzzer_build(self.test_files_path)) + self.assertFalse( + build_fuzzers.check_fuzzer_build(self.test_files_path, self.SANITIZER, + self.LANGUAGE)) @mock.patch('helper.docker_run') def test_allow_broken_fuzz_targets_percentage(self, mocked_docker_run): """Tests that ALLOWED_BROKEN_TARGETS_PERCENTAGE is set when running docker if passed to check_fuzzer_build.""" mocked_docker_run.return_value = 0 - test_fuzzer_dir = os.path.join(TEST_FILES_PATH, 'out') + test_fuzzer_dir = os.path.join(TEST_DATA_PATH, 'out') build_fuzzers.check_fuzzer_build(test_fuzzer_dir, + self.SANITIZER, + self.LANGUAGE, allowed_broken_targets_percentage='0') self.assertIn('-e ALLOWED_BROKEN_TARGETS_PERCENTAGE=0', ' '.join(mocked_docker_run.call_args[0][0])) diff --git a/infra/cifuzz/cifuzz-base/Dockerfile b/infra/cifuzz/cifuzz-base/Dockerfile index 0aee3b2cf..e0599dbbe 100644 --- a/infra/cifuzz/cifuzz-base/Dockerfile +++ b/infra/cifuzz/cifuzz-base/Dockerfile @@ -14,25 +14,19 @@ # ################################################################################ +# Don't bother with a slimmer base image. +# When we pull base-builder to build project builder image we need to pull +# ubuntu:16.04 anyway. So in the long run we probably would waste time if +# we pulled something like alpine here instead. FROM ubuntu:16.04 -RUN apt-get update && apt-get install -y git \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg2 \ - software-properties-common \ - python3 +RUN apt-get update && \ + apt-get install ca-certificates wget python3 git-core --no-install-recommends -y && \ + wget https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce-cli_20.10.5~3-0~ubuntu-xenial_amd64.deb -O /tmp/docker-ce.deb && \ + dpkg -i /tmp/docker-ce.deb && rm /tmp/docker-ce.deb && \ + apt-get remove wget -y --purge -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && apt-key fingerprint 0EBFCD88 -RUN add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ - xenial \ - stable" - -RUN apt-get update && apt-get install docker-ce docker-ce-cli containerd.io -y - ENV OSS_FUZZ_ROOT=/opt/oss-fuzz ADD . ${OSS_FUZZ_ROOT} RUN rm -rf ${OSS_FUZZ_ROOT}/infra
\ No newline at end of file diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py index fd1871497..ad2cd36c6 100644 --- a/infra/cifuzz/config_utils.py +++ b/infra/cifuzz/config_utils.py @@ -18,14 +18,16 @@ import enum import os import json +import environment + def _get_project_repo_name(): - return os.path.basename(os.getenv('GITHUB_REPOSITORY', '')) + return os.path.basename(environment.get('GITHUB_REPOSITORY', '')) def _get_pr_ref(event): if event == 'pull_request': - return os.getenv('GITHUB_REF') + return environment.get('GITHUB_REF') return None @@ -40,7 +42,7 @@ def _get_project_name(): def _is_dry_run(): """Returns True if configured to do a dry run.""" - return os.getenv('DRY_RUN', 'false').lower() == 'true' + return environment.get_bool('DRY_RUN', 'false') def get_project_src_path(workspace): @@ -62,6 +64,19 @@ def get_project_src_path(workspace): return os.path.join(workspace, path) +DEFAULT_LANGUAGE = 'c++' + + +def _get_language(): + """Returns the project language.""" + # Get language from environment. We took this approach because the convenience + # given to OSS-Fuzz users by not making them specify the language again (and + # getting it from the project.yaml) is outweighed by the complexity in + # implementing this. A lot of the complexity comes from our unittests not + # setting a proper projet at this point. + return os.getenv('LANGUAGE', DEFAULT_LANGUAGE) + + # pylint: disable=too-few-public-methods,too-many-instance-attributes @@ -81,14 +96,22 @@ class BaseConfig: self.dry_run = _is_dry_run() self.sanitizer = _get_sanitizer() self.build_integration_path = os.getenv('BUILD_INTEGRATION_PATH') + self.language = _get_language() event_path = os.getenv('GITHUB_EVENT_PATH') self.is_github = bool(event_path) logging.debug('Is github: %s.', self.is_github) + # TODO(metzman): Parse env like we do in ClusterFuzz. + self.low_disk_space = environment.get('LOW_DISK_SPACE', False) + + @property + def is_internal(self): + """Returns True if this is an OSS-Fuzz project.""" + return not self.build_integration_path @property def platform(self): """Returns the platform CIFuzz is runnning on.""" - if self.build_integration_path: + if not self.is_internal: return self.Platform.EXTERNAL_GITHUB if self.is_github: return self.Platform.INTERNAL_GITHUB @@ -149,6 +172,7 @@ class BuildFuzzersConfig(BaseConfig): self.allowed_broken_targets_percentage = os.getenv( 'ALLOWED_BROKEN_TARGETS_PERCENTAGE') + self.bad_build_check = environment.get_bool('BAD_BUILD_CHECK', 'true') # TODO(metzman): Use better system for interpreting env vars. What if env # var is set to '0'? diff --git a/infra/cifuzz/config_utils_test.py b/infra/cifuzz/config_utils_test.py index 71e7450fa..6f87bd4c5 100644 --- a/infra/cifuzz/config_utils_test.py +++ b/infra/cifuzz/config_utils_test.py @@ -13,19 +13,38 @@ # limitations under the License. """Module for getting the configuration CIFuzz needs to run.""" import os -import sys import unittest import config_utils - -# pylint: disable=wrong-import-position,import-error -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - import test_helpers # pylint: disable=no-self-use +class BaseConfigTest(unittest.TestCase): + """Tests for BaseConfig.""" + + def setUp(self): + test_helpers.patch_environ(self) + + def _create_config(self): + return config_utils.BuildFuzzersConfig() + + def test_language_default(self): + """Tests that the correct default language is set.""" + os.environ['BUILD_INTEGRATION_PATH'] = '/path' + config = self._create_config() + self.assertEqual(config.language, 'c++') + + def test_language(self): + """Tests that the correct language is set.""" + os.environ['BUILD_INTEGRATION_PATH'] = '/path' + language = 'python' + os.environ['LANGUAGE'] = language + config = self._create_config() + self.assertEqual(config.language, language) + + class BuildFuzzersConfigTest(unittest.TestCase): """Tests for BuildFuzzersConfig.""" diff --git a/infra/cifuzz/coverage.py b/infra/cifuzz/coverage.py index b5c6fbf1a..9a179c59d 100644 --- a/infra/cifuzz/coverage.py +++ b/infra/cifuzz/coverage.py @@ -115,7 +115,7 @@ def _get_latest_cov_report_info(project_name): LATEST_REPORT_INFO_PATH, project_name + '.json') latest_cov_info = get_json_from_url(latest_report_info_url) - if not latest_cov_info is None: + if latest_cov_info is None: logging.error('Could not get the coverage report json from url: %s.', latest_report_info_url) return None diff --git a/infra/cifuzz/coverage_test.py b/infra/cifuzz/coverage_test.py index 57120f5f5..1b24d798c 100644 --- a/infra/cifuzz/coverage_test.py +++ b/infra/cifuzz/coverage_test.py @@ -21,8 +21,8 @@ import coverage # pylint: disable=protected-access -TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files') +TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_data') PROJECT_NAME = 'curl' REPO_PATH = '/src/curl' @@ -31,7 +31,7 @@ PROJECT_COV_JSON_FILENAME = 'example_curl_cov.json' FUZZ_TARGET_COV_JSON_FILENAME = 'example_curl_fuzzer_cov.json' INVALID_TARGET = 'not-a-fuzz-target' -with open(os.path.join(TEST_FILES_PATH, +with open(os.path.join(TEST_DATA_PATH, PROJECT_COV_JSON_FILENAME),) as cov_file_handle: PROJECT_COV_INFO = json.loads(cov_file_handle.read()) @@ -39,19 +39,28 @@ with open(os.path.join(TEST_FILES_PATH, class GetFuzzerStatsDirUrlTest(unittest.TestCase): """Tests _get_fuzzer_stats_dir_url.""" - @mock.patch('coverage.get_json_from_url', return_value={}) + @mock.patch('coverage.get_json_from_url', + return_value={ + 'fuzzer_stats_dir': + 'gs://oss-fuzz-coverage/systemd/fuzzer_stats/20210303' + }) def test_get_valid_project(self, mocked_get_json_from_url): """Tests that a project's coverage report can be downloaded and parsed. NOTE: This test relies on the PROJECT_NAME repo's coverage report. The "example" project was not used because it has no coverage reports. """ - coverage._get_fuzzer_stats_dir_url(PROJECT_NAME) + result = coverage._get_fuzzer_stats_dir_url(PROJECT_NAME) (url,), _ = mocked_get_json_from_url.call_args self.assertEqual( 'https://storage.googleapis.com/oss-fuzz-coverage/' 'latest_report_info/curl.json', url) + expected_result = ( + 'https://storage.googleapis.com/oss-fuzz-coverage/systemd/fuzzer_stats/' + '20210303') + self.assertEqual(result, expected_result) + def test_get_invalid_project(self): """Tests that passing a bad project returns None.""" self.assertIsNone(coverage._get_fuzzer_stats_dir_url('not-a-proj')) @@ -98,7 +107,7 @@ class GetFilesCoveredByTargetTest(unittest.TestCase): def test_valid_target(self): """Tests that covered files can be retrieved from a coverage report.""" - with open(os.path.join(TEST_FILES_PATH, + with open(os.path.join(TEST_DATA_PATH, FUZZ_TARGET_COV_JSON_FILENAME),) as file_handle: fuzzer_cov_info = json.loads(file_handle.read()) @@ -106,7 +115,7 @@ class GetFilesCoveredByTargetTest(unittest.TestCase): return_value=fuzzer_cov_info): file_list = self.coverage_getter.get_files_covered_by_target(FUZZ_TARGET) - curl_files_list_path = os.path.join(TEST_FILES_PATH, + curl_files_list_path = os.path.join(TEST_DATA_PATH, 'example_curl_file_list.json') with open(curl_files_list_path) as file_handle: expected_file_list = json.loads(file_handle.read()) @@ -152,5 +161,34 @@ class IsFileCoveredTest(unittest.TestCase): self.assertFalse(coverage.is_file_covered(file_coverage)) +class GetLatestCovReportInfo(unittest.TestCase): + """Tests that _get_latest_cov_report_info works as intended.""" + + PROJECT = 'project' + LATEST_REPORT_INFO_URL = ('https://storage.googleapis.com/oss-fuzz-coverage/' + 'latest_report_info/project.json') + + @mock.patch('logging.error') + @mock.patch('coverage.get_json_from_url', return_value={'coverage': 1}) + def test_get_latest_cov_report_info(self, mocked_get_json_from_url, + mocked_error): + """Tests that _get_latest_cov_report_info works as intended.""" + result = coverage._get_latest_cov_report_info(self.PROJECT) + self.assertEqual(result, {'coverage': 1}) + mocked_error.assert_not_called() + mocked_get_json_from_url.assert_called_with(self.LATEST_REPORT_INFO_URL) + + @mock.patch('logging.error') + @mock.patch('coverage.get_json_from_url', return_value=None) + def test_get_latest_cov_report_info_fail(self, _, mocked_error): + """Tests that _get_latest_cov_report_info works as intended when we can't + get latest report info.""" + result = coverage._get_latest_cov_report_info('project') + self.assertIsNone(result) + mocked_error.assert_called_with( + 'Could not get the coverage report json from url: %s.', + self.LATEST_REPORT_INFO_URL) + + if __name__ == '__main__': unittest.main() diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py new file mode 100644 index 000000000..eb993e28d --- /dev/null +++ b/infra/cifuzz/docker.py @@ -0,0 +1,38 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for dealing with docker.""" +import os +import sys + +# pylint: disable=wrong-import-position,import-error +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import utils + +BASE_BUILDER_TAG = 'gcr.io/oss-fuzz-base/base-builder' +BASE_RUNNER_TAG = 'gcr.io/oss-fuzz-base/base-runner' +MSAN_LIBS_BUILDER_TAG = 'gcr.io/oss-fuzz-base/msan-libs-builder' +PROJECT_TAG_PREFIX = 'gcr.io/oss-fuzz/' + + +def get_project_image_name(project): + """Returns the name of the project builder image for |project_name|.""" + return PROJECT_TAG_PREFIX + project + + +def delete_images(images): + """Deletes |images|.""" + command = ['docker', 'rmi', '-f'] + images + utils.execute(command) + utils.execute(['docker', 'builder', 'prune', '-f']) diff --git a/infra/cifuzz/environment.py b/infra/cifuzz/environment.py new file mode 100644 index 000000000..4cc0f846b --- /dev/null +++ b/infra/cifuzz/environment.py @@ -0,0 +1,54 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Module for dealing with env vars.""" + +import ast +import os + + +def _eval_value(value_string): + """Returns evaluated value.""" + try: + return ast.literal_eval(value_string) + except: # pylint: disable=bare-except + # String fallback. + return value_string + + +def get(env_var, default_value=None): + """Returns an environment variable value.""" + value_string = os.getenv(env_var) + if value_string is None: + return default_value + + return _eval_value(value_string) + + +def get_bool(env_var, default_value=None): + """Returns a boolean environment variable value. This is needed because a lot + of CIFuzz users specified 'false' for dry-run. So we need to special case + this.""" + value = get(env_var, default_value) + if not isinstance(value, str): + return bool(value) + + lower_value = value.lower() + allowed_values = {'true', 'false'} + if lower_value not in allowed_values: + raise Exception(('Bool env var {env_var} value {value} is invalid. ' + 'Must be one of {allowed_values}').format( + env_var=env_var, + value=value, + allowed_values=allowed_values)) + return lower_value == 'true' diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py index 7bccfa4e1..c623bf60d 100644 --- a/infra/cifuzz/fuzz_target.py +++ b/infra/cifuzz/fuzz_target.py @@ -16,10 +16,13 @@ import collections import logging import os import re +import shutil import stat import subprocess import sys +import docker + # pylint: disable=wrong-import-position,import-error sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import utils @@ -28,6 +31,8 @@ logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.DEBUG) +# Use a fixed seed for determinism. Use len_control=0 since we don't have enough +# time fuzzing for len_control to make sense (probably). LIBFUZZER_OPTIONS = '-seed=1337 -len_control=0' # The number of reproduce attempts for a crash. @@ -78,6 +83,7 @@ class FuzzTarget: self.out_dir = out_dir self.clusterfuzz_deployment = clusterfuzz_deployment self.config = config + self.latest_corpus_path = None def fuzz(self): """Starts the fuzz target run for the length of time specified by duration. @@ -98,8 +104,7 @@ class FuzzTarget: command += [ '-e', 'FUZZING_ENGINE=libfuzzer', '-e', 'SANITIZER=' + self.config.sanitizer, '-e', 'CIFUZZ=True', '-e', - 'RUN_FUZZER_MODE=interactive', 'gcr.io/oss-fuzz-base/base-runner', - 'bash', '-c' + 'RUN_FUZZER_MODE=interactive', docker.BASE_RUNNER_TAG, 'bash', '-c' ] run_fuzzer_command = 'run_fuzzer {fuzz_target} {options}'.format( @@ -107,10 +112,10 @@ class FuzzTarget: options=LIBFUZZER_OPTIONS + ' -max_total_time=' + str(self.duration)) # If corpus can be downloaded use it for fuzzing. - latest_corpus_path = self.clusterfuzz_deployment.download_corpus( + self.latest_corpus_path = self.clusterfuzz_deployment.download_corpus( self.target_name, self.out_dir) - if latest_corpus_path: - run_fuzzer_command = run_fuzzer_command + ' ' + latest_corpus_path + if self.latest_corpus_path: + run_fuzzer_command = run_fuzzer_command + ' ' + self.latest_corpus_path command.append(run_fuzzer_command) logging.info('Running command: %s', ' '.join(command)) @@ -136,10 +141,37 @@ class FuzzTarget: if not testcase: logging.error(b'No testcase found in stacktrace: %s.', stderr) return FuzzResult(None, None) + + utils.binary_print(b'Fuzzer: %s. Detected bug:\n%s' % + (self.target_name.encode(), stderr)) if self.is_crash_reportable(testcase): + # We found a bug in the fuzz target and we will report it. return FuzzResult(testcase, stderr) + + # We found a bug but we won't report it. return FuzzResult(None, None) + def free_disk_if_needed(self): + """Deletes things that are no longer needed from fuzzing this fuzz target to + save disk space if needed.""" + if not self.config.low_disk_space: + return + logging.info( + 'Deleting corpus, seed corpus and fuzz target of %s to save disk.', + self.target_name) + + # Delete the seed corpus, corpus, and fuzz target. + if self.latest_corpus_path and os.path.exists(self.latest_corpus_path): + # Use ignore_errors=True to fix + # https://github.com/google/oss-fuzz/issues/5383. + shutil.rmtree(self.latest_corpus_path, ignore_errors=True) + + os.remove(self.target_path) + target_seed_corpus_path = self.target_path + '_seed_corpus.zip' + if os.path.exists(target_seed_corpus_path): + os.remove(target_seed_corpus_path) + logging.info('Done deleting.') + def is_reproducible(self, testcase, target_path): """Checks if the testcase reproduces. @@ -176,8 +208,7 @@ class FuzzTarget: ] command += [ - '-t', 'gcr.io/oss-fuzz-base/base-runner', 'reproduce', self.target_name, - '-runs=100' + '-t', docker.BASE_RUNNER_TAG, 'reproduce', self.target_name, '-runs=100' ] logging.info('Running reproduce command: %s.', ' '.join(command)) @@ -246,7 +277,6 @@ class FuzzTarget: logging.info('The crash is reproducible. The crash doesn\'t reproduce ' 'on old builds. This code change probably introduced the ' 'crash.') - return True logging.info('The crash is reproducible on old builds ' diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py index 8a506fa59..8bec234dc 100644 --- a/infra/cifuzz/fuzz_target_test.py +++ b/infra/cifuzz/fuzz_target_test.py @@ -148,8 +148,7 @@ class GetTestCaseTest(unittest.TestCase): def test_valid_error_string(self): """Tests that get_testcase returns the correct testcase give an error.""" testcase_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files', - 'example_crash_fuzzer_output.txt') + 'test_data', 'example_crash_fuzzer_output.txt') with open(testcase_path, 'rb') as test_fuzz_output: parsed_testcase = self.test_target.get_testcase(test_fuzz_output.read()) self.assertEqual( diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py index 2a2a89e5f..513cfb6fa 100644 --- a/infra/cifuzz/run_fuzzers.py +++ b/infra/cifuzz/run_fuzzers.py @@ -91,7 +91,9 @@ class BaseFuzzTargetRunner: """Fuzzes with |fuzz_target_obj| and returns the result.""" # TODO(metzman): Make children implement this so that the batch runner can # do things differently. - return fuzz_target_obj.fuzz() + result = fuzz_target_obj.fuzz() + fuzz_target_obj.free_disk_if_needed() + return result @property def quit_on_bug_found(self): @@ -100,9 +102,12 @@ class BaseFuzzTargetRunner: raise NotImplementedError('Child class must implement method') def get_fuzz_target_artifact(self, target, artifact_name): - """Returns the path of a fuzzing |artifact| named |artifact_name| for - |target|.""" - artifact_name = target.target_name + '-' + artifact_name + """Returns the path of a fuzzing artifact named |artifact_name| for + |fuzz_target|.""" + artifact_name = '{target_name}-{sanitizer}-{artifact_name}'.format( + target_name=target.target_name, + sanitizer=self.config.sanitizer, + artifact_name=artifact_name) return os.path.join(self.artifacts_dir, artifact_name) def create_fuzz_target_obj(self, target_path, run_seconds): @@ -140,12 +145,9 @@ class BaseFuzzTargetRunner: target.target_name) continue - # We found a bug in the fuzz target. - utils.binary_print(b'Fuzzer: %s. Detected bug:\n%s' % - (target.target_name.encode(), result.stacktrace)) - # TODO(metzman): Do this with filestore. - testcase_artifact_path = self.get_fuzz_target_artifact(target, 'testcase') + testcase_artifact_path = self.get_fuzz_target_artifact( + target, os.path.basename(result.testcase)) shutil.move(result.testcase, testcase_artifact_path) bug_summary_artifact_path = self.get_fuzz_target_artifact( target, 'bug-summary.txt') diff --git a/infra/cifuzz/run_fuzzers_entrypoint.py b/infra/cifuzz/run_fuzzers_entrypoint.py index f810e38f8..46e208dc0 100644 --- a/infra/cifuzz/run_fuzzers_entrypoint.py +++ b/infra/cifuzz/run_fuzzers_entrypoint.py @@ -11,11 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Runs specific OSS-Fuzz project's fuzzers for CI tools.""" +"""Runs a specific OSS-Fuzz project's fuzzers for CI tools.""" import logging import sys import config_utils +import docker import run_fuzzers # pylint: disable=c-extension-no-member @@ -26,6 +27,21 @@ logging.basicConfig( level=logging.DEBUG) +def delete_unneeded_docker_images(config): + """Deletes unneeded docker images if running in an environment with low + disk space.""" + if not config.low_disk_space: + return + logging.info('Deleting builder docker images to save disk space.') + project_image = docker.get_project_image_name(config.project_name) + images = [ + project_image, + docker.BASE_RUNNER_TAG, + docker.MSAN_LIBS_BUILDER_TAG, + ] + docker.delete_images(images) + + def main(): """Runs OSS-Fuzz project's fuzzers for CI tools. This is the entrypoint for the run_fuzzers github action. @@ -62,6 +78,7 @@ def main(): logging.error('This script needs to be run within Github actions.') return returncode + delete_unneeded_docker_images(config) # Run the specified project's fuzzers from the build. result = run_fuzzers.run_fuzzers(config) if result == run_fuzzers.RunFuzzersResult.ERROR: diff --git a/infra/cifuzz/run_fuzzers_test.py b/infra/cifuzz/run_fuzzers_test.py index 847ddf399..b2659903c 100644 --- a/infra/cifuzz/run_fuzzers_test.py +++ b/infra/cifuzz/run_fuzzers_test.py @@ -37,13 +37,13 @@ import test_helpers EXAMPLE_PROJECT = 'example' # Location of files used for testing. -TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files') +TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_data') -MEMORY_FUZZER_DIR = os.path.join(TEST_FILES_PATH, 'memory') +MEMORY_FUZZER_DIR = os.path.join(TEST_DATA_PATH, 'memory') MEMORY_FUZZER = 'curl_fuzzer_memory' -UNDEFINED_FUZZER_DIR = os.path.join(TEST_FILES_PATH, 'undefined') +UNDEFINED_FUZZER_DIR = os.path.join(TEST_DATA_PATH, 'undefined') UNDEFINED_FUZZER = 'curl_fuzzer_undefined' FUZZ_SECONDS = 10 @@ -227,7 +227,8 @@ class BaseFuzzTargetRunnerTest(unittest.TestCase): target.target_name = target_name fuzz_target_artifact = runner.get_fuzz_target_artifact( target, artifact_name) - expected_fuzz_target_artifact = 'artifacts-dir/target_name-artifact-name' + expected_fuzz_target_artifact = ( + 'artifacts-dir/target_name-address-artifact-name') self.assertEqual(fuzz_target_artifact, expected_fuzz_target_artifact) @@ -263,7 +264,7 @@ class CiFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): magic_mock.target_name = 'target1' mocked_create_fuzz_target_obj.return_value = magic_mock self.assertTrue(runner.run_fuzz_targets()) - self.assertIn('target1-testcase', os.listdir(runner.artifacts_dir)) + self.assertIn('target1-address-testcase', os.listdir(runner.artifacts_dir)) self.assertEqual(mocked_run_fuzz_target.call_count, 1) @@ -279,7 +280,7 @@ class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): def test_run_fuzz_targets_quits(self, mocked_create_fuzz_target_obj, mocked_run_fuzz_target, mocked_get_fuzz_targets): - """Tests that run_fuzz_targets quits on the first crash it finds.""" + """Tests that run_fuzz_targets doesn't quit on the first crash it finds.""" workspace = 'workspace' out_path = os.path.join(workspace, 'out') self.fs.create_dir(out_path) @@ -290,8 +291,8 @@ class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): mocked_get_fuzz_targets.return_value = ['target1', 'target2'] runner.initialize() - testcase1 = os.path.join(workspace, 'testcase1') - testcase2 = os.path.join(workspace, 'testcase2') + testcase1 = os.path.join(workspace, 'testcase-aaa') + testcase2 = os.path.join(workspace, 'testcase-bbb') self.fs.create_file(testcase1) self.fs.create_file(testcase2) stacktrace = b'stacktrace' @@ -312,7 +313,8 @@ class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase): magic_mock.target_name = 'target1' mocked_create_fuzz_target_obj.return_value = magic_mock self.assertTrue(runner.run_fuzz_targets()) - self.assertIn('target1-testcase', os.listdir(runner.artifacts_dir)) + self.assertIn('target1-address-testcase-aaa', + os.listdir(runner.artifacts_dir)) self.assertEqual(mocked_run_fuzz_target.call_count, 2) @@ -333,7 +335,7 @@ class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin, side_effect=[True, False]): with tempfile.TemporaryDirectory() as tmp_dir: workspace = os.path.join(tmp_dir, 'workspace') - shutil.copytree(TEST_FILES_PATH, workspace) + shutil.copytree(TEST_DATA_PATH, workspace) config = _create_config(fuzz_seconds=FUZZ_SECONDS, workspace=workspace, project_name=EXAMPLE_PROJECT) @@ -349,17 +351,17 @@ class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin, def test_old_bug_found(self, _): """Tests run_fuzzers with a bug found in OSS-Fuzz before.""" config = _create_config(fuzz_seconds=FUZZ_SECONDS, - workspace=TEST_FILES_PATH, + workspace=TEST_DATA_PATH, project_name=EXAMPLE_PROJECT) with tempfile.TemporaryDirectory() as tmp_dir: workspace = os.path.join(tmp_dir, 'workspace') - shutil.copytree(TEST_FILES_PATH, workspace) + shutil.copytree(TEST_DATA_PATH, workspace) config = _create_config(fuzz_seconds=FUZZ_SECONDS, - workspace=TEST_FILES_PATH, + workspace=TEST_DATA_PATH, project_name=EXAMPLE_PROJECT) result = run_fuzzers.run_fuzzers(config) self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND) - build_dir = os.path.join(TEST_FILES_PATH, 'out', self.BUILD_DIR_NAME) + build_dir = os.path.join(TEST_DATA_PATH, 'out', self.BUILD_DIR_NAME) self.assertTrue(os.path.exists(build_dir)) self.assertNotEqual(0, len(os.listdir(build_dir))) diff --git a/infra/cifuzz/stack_parser.py b/infra/cifuzz/stack_parser.py index 0077caae9..69c44bc2e 100644 --- a/infra/cifuzz/stack_parser.py +++ b/infra/cifuzz/stack_parser.py @@ -13,6 +13,8 @@ # limitations under the License. """Module for parsing stacks from fuzz targets.""" +import logging + # From clusterfuzz: src/python/crash_analysis/crash_analyzer.py # Used to get the beginning of the stacktrace. STACKTRACE_TOOL_MARKERS = [ @@ -51,25 +53,33 @@ def parse_fuzzer_output(fuzzer_output, parsed_output_file_path): parsed_output_file_path: The location to store the parsed output. """ # Get index of key file points. + begin_stack = None for marker in STACKTRACE_TOOL_MARKERS: marker_index = fuzzer_output.find(marker) - if marker_index: + if marker_index != -1: begin_stack = marker_index break - end_stack = -1 + if begin_stack is None: + logging.error( + b'Could not find a begin stack marker (%s) in fuzzer output:\n%s', + STACKTRACE_TOOL_MARKERS, fuzzer_output) + return + + end_stack = None for marker in STACKTRACE_END_MARKERS: marker_index = fuzzer_output.find(marker) - if marker_index: + if marker_index != -1: end_stack = marker_index + len(marker) break - if begin_stack is None or end_stack is None: + if end_stack is None: + logging.error( + b'Could not find an end stack marker (%s) in fuzzer output:\n%s', + STACKTRACE_END_MARKERS, fuzzer_output) return summary_str = fuzzer_output[begin_stack:end_stack] - if not summary_str: - return # Write sections of fuzzer output to specific files. with open(parsed_output_file_path, 'ab') as summary_handle: diff --git a/infra/cifuzz/stack_parser_test.py b/infra/cifuzz/stack_parser_test.py index 9b05710fc..faf601fd5 100644 --- a/infra/cifuzz/stack_parser_test.py +++ b/infra/cifuzz/stack_parser_test.py @@ -14,7 +14,9 @@ """Tests for stack_parser.""" import os import unittest +from unittest import mock +import parameterized from pyfakefs import fake_filesystem_unittest import stack_parser @@ -23,9 +25,9 @@ import stack_parser # https://github.com/google/oss-fuzz/tree/master/projects/example project. EXAMPLE_PROJECT = 'example' -# Location of files used for testing. -TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'test_files') +# Location of data used for testing. +TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'test_data') class ParseOutputTest(fake_filesystem_unittest.TestCase): @@ -33,33 +35,42 @@ class ParseOutputTest(fake_filesystem_unittest.TestCase): def setUp(self): self.setUpPyfakefs() + self.maxDiff = None # pylint: disable=invalid-name - def test_parse_valid_output(self): + @parameterized.parameterized.expand([('example_crash_fuzzer_output.txt', + 'example_crash_fuzzer_bug_summary.txt'), + ('msan_crash_fuzzer_output.txt', + 'msan_crash_fuzzer_bug_summary.txt')]) + def test_parse_valid_output(self, fuzzer_output_file, bug_summary_file): """Checks that the parse fuzzer output can correctly parse output.""" # Read the fuzzer output from disk. - fuzzer_output_path = os.path.join(TEST_FILES_PATH, - 'example_crash_fuzzer_output.txt') + fuzzer_output_path = os.path.join(TEST_DATA_PATH, fuzzer_output_file) self.fs.add_real_file(fuzzer_output_path) with open(fuzzer_output_path, 'rb') as fuzzer_output_handle: fuzzer_output = fuzzer_output_handle.read() bug_summary_path = '/bug-summary.txt' - stack_parser.parse_fuzzer_output(fuzzer_output, bug_summary_path) + with mock.patch('logging.info') as mocked_info: + stack_parser.parse_fuzzer_output(fuzzer_output, bug_summary_path) + mocked_info.assert_not_called() + with open(bug_summary_path) as bug_summary_handle: bug_summary = bug_summary_handle.read() # Compare the bug to the expected one. - expected_bug_summary_path = os.path.join(TEST_FILES_PATH, - 'bug_summary_example.txt') + expected_bug_summary_path = os.path.join(TEST_DATA_PATH, bug_summary_file) self.fs.add_real_file(expected_bug_summary_path) with open(expected_bug_summary_path) as expected_bug_summary_handle: expected_bug_summary = expected_bug_summary_handle.read() + self.assertEqual(expected_bug_summary, bug_summary) def test_parse_invalid_output(self): """Checks that no files are created when an invalid input was given.""" artifact_path = '/bug-summary.txt' - stack_parser.parse_fuzzer_output(b'not a valid output_string', - artifact_path) + with mock.patch('logging.error') as mocked_error: + stack_parser.parse_fuzzer_output(b'not a valid output_string', + artifact_path) + assert mocked_error.call_count self.assertFalse(os.path.exists(artifact_path)) diff --git a/infra/cifuzz/test_files/bug_summary_example.txt b/infra/cifuzz/test_data/example_crash_fuzzer_bug_summary.txt index 8caebad0c..8caebad0c 100644 --- a/infra/cifuzz/test_files/bug_summary_example.txt +++ b/infra/cifuzz/test_data/example_crash_fuzzer_bug_summary.txt diff --git a/infra/cifuzz/test_files/example_crash_fuzzer_output.txt b/infra/cifuzz/test_data/example_crash_fuzzer_output.txt index d316f5f40..d316f5f40 100644 --- a/infra/cifuzz/test_files/example_crash_fuzzer_output.txt +++ b/infra/cifuzz/test_data/example_crash_fuzzer_output.txt diff --git a/infra/cifuzz/test_files/example_curl_cov.json b/infra/cifuzz/test_data/example_curl_cov.json index 0936102fd..0936102fd 100644 --- a/infra/cifuzz/test_files/example_curl_cov.json +++ b/infra/cifuzz/test_data/example_curl_cov.json diff --git a/infra/cifuzz/test_files/example_curl_file_list.json b/infra/cifuzz/test_data/example_curl_file_list.json index 0ed1965c5..0ed1965c5 100644 --- a/infra/cifuzz/test_files/example_curl_file_list.json +++ b/infra/cifuzz/test_data/example_curl_file_list.json diff --git a/infra/cifuzz/test_files/example_curl_fuzzer_cov.json b/infra/cifuzz/test_data/example_curl_fuzzer_cov.json index 6f8c2498c..6f8c2498c 100644 --- a/infra/cifuzz/test_files/example_curl_fuzzer_cov.json +++ b/infra/cifuzz/test_data/example_curl_fuzzer_cov.json diff --git a/infra/cifuzz/test_files/external-project/Makefile b/infra/cifuzz/test_data/external-project/Makefile index 2c1773776..2c1773776 100644 --- a/infra/cifuzz/test_files/external-project/Makefile +++ b/infra/cifuzz/test_data/external-project/Makefile diff --git a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.cpp b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.cpp index 71fa8cae2..71fa8cae2 100644 --- a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.cpp +++ b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.cpp diff --git a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.dict b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.dict index 224679bf4..224679bf4 100644 --- a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.dict +++ b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.dict diff --git a/infra/cifuzz/test_files/external-project/my_api.cpp b/infra/cifuzz/test_data/external-project/my_api.cpp index 9a2c1bc1c..9a2c1bc1c 100644 --- a/infra/cifuzz/test_files/external-project/my_api.cpp +++ b/infra/cifuzz/test_data/external-project/my_api.cpp diff --git a/infra/cifuzz/test_files/external-project/my_api.h b/infra/cifuzz/test_data/external-project/my_api.h index 325aa15cc..325aa15cc 100644 --- a/infra/cifuzz/test_files/external-project/my_api.h +++ b/infra/cifuzz/test_data/external-project/my_api.h diff --git a/infra/cifuzz/test_files/external-project/oss-fuzz/Dockerfile b/infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile index e9dc33031..e9dc33031 100644 --- a/infra/cifuzz/test_files/external-project/oss-fuzz/Dockerfile +++ b/infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile diff --git a/infra/cifuzz/test_files/external-project/oss-fuzz/build.sh b/infra/cifuzz/test_data/external-project/oss-fuzz/build.sh index 2c52ef90f..2c52ef90f 100644 --- a/infra/cifuzz/test_files/external-project/oss-fuzz/build.sh +++ b/infra/cifuzz/test_data/external-project/oss-fuzz/build.sh diff --git a/infra/cifuzz/test_files/external-project/standalone_fuzz_target_runner.cpp b/infra/cifuzz/test_data/external-project/standalone_fuzz_target_runner.cpp index 38a0454f0..38a0454f0 100644 --- a/infra/cifuzz/test_files/external-project/standalone_fuzz_target_runner.cpp +++ b/infra/cifuzz/test_data/external-project/standalone_fuzz_target_runner.cpp diff --git a/infra/cifuzz/test_files/memory/out/curl_fuzzer_memory b/infra/cifuzz/test_data/memory/out/curl_fuzzer_memory Binary files differindex c602ce970..c602ce970 100755 --- a/infra/cifuzz/test_files/memory/out/curl_fuzzer_memory +++ b/infra/cifuzz/test_data/memory/out/curl_fuzzer_memory diff --git a/infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt b/infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt new file mode 100644 index 000000000..b55e9c6b7 --- /dev/null +++ b/infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt @@ -0,0 +1,22 @@ +MemorySanitizer: use-of-uninitialized-value +#0 0x52675f in LLVMFuzzerTestOneInput /src/cifuzz-example/do_stuff_fuzzer.cpp:13:7 +#1 0x45a431 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:599:15 +#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3 +#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3 +#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6 +#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10 +#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f) +#7 0x41eab8 in _start (out/do_stuff_fuzzer+0x41eab8) + +DEDUP_TOKEN: LLVMFuzzerTestOneInput--fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) +Uninitialized value was created by a heap allocation +#0 0x4d57ad in malloc /src/llvm-project/compiler-rt/lib/msan/msan_interceptors.cpp:901:3 +#1 0x437c07 in operator new(unsigned long) (out/do_stuff_fuzzer+0x437c07) +#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3 +#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3 +#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6 +#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10 +#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f) +DEDUP_TOKEN: malloc--operator new(unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) + +SUMMARY:
\ No newline at end of file diff --git a/infra/cifuzz/test_data/msan_crash_fuzzer_output.txt b/infra/cifuzz/test_data/msan_crash_fuzzer_output.txt new file mode 100644 index 000000000..c803bfb1c --- /dev/null +++ b/infra/cifuzz/test_data/msan_crash_fuzzer_output.txt @@ -0,0 +1,39 @@ +Dictionary: 3 entries +INFO: Running with entropic power schedule (0xFF, 100). +INFO: Seed: 1337 +INFO: Loaded 1 modules (184 inline 8-bit counters): 184 [0x829300, 0x8293b8), +INFO: Loaded 1 PC tables (184 PCs): 184 [0x5dc910,0x5dd490), +INFO: 5 files found in /tmp/do_stuff_fuzzer_corpus +INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes +==13==WARNING: MemorySanitizer: use-of-uninitialized-value +#0 0x52675f in LLVMFuzzerTestOneInput /src/cifuzz-example/do_stuff_fuzzer.cpp:13:7 +#1 0x45a431 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:599:15 +#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3 +#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3 +#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6 +#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10 +#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f) +#7 0x41eab8 in _start (out/do_stuff_fuzzer+0x41eab8) + +DEDUP_TOKEN: LLVMFuzzerTestOneInput--fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) +Uninitialized value was created by a heap allocation +#0 0x4d57ad in malloc /src/llvm-project/compiler-rt/lib/msan/msan_interceptors.cpp:901:3 +#1 0x437c07 in operator new(unsigned long) (out/do_stuff_fuzzer+0x437c07) +#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3 +#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3 +#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6 +#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10 +#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f) +DEDUP_TOKEN: malloc--operator new(unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) + +SUMMARY: MemorySanitizer: use-of-uninitialized-value /src/cifuzz-example/do_stuff_fuzzer.cpp:13:7 in LLVMFuzzerTestOneInput +Unique heap origins: 65 +Stack depot allocated bytes: 4424 +Unique origin histories: 29 +History depot allocated bytes: 696 +Exiting +MS: 0 ; base unit: 0000000000000000000000000000000000000000 + + +artifact_prefix='./'; Test unit written to ./crash-da39a3ee5e6b4b0d3255bfef95601890afd80709 +Base64: diff --git a/infra/cifuzz/test_files/out/example_crash_fuzzer b/infra/cifuzz/test_data/out/example_crash_fuzzer Binary files differindex 704800dda..704800dda 100755 --- a/infra/cifuzz/test_files/out/example_crash_fuzzer +++ b/infra/cifuzz/test_data/out/example_crash_fuzzer diff --git a/infra/cifuzz/test_files/out/example_nocrash_fuzzer b/infra/cifuzz/test_data/out/example_nocrash_fuzzer Binary files differindex e4ff86042..e4ff86042 100755 --- a/infra/cifuzz/test_files/out/example_nocrash_fuzzer +++ b/infra/cifuzz/test_data/out/example_nocrash_fuzzer diff --git a/infra/cifuzz/test_files/undefined/out/curl_fuzzer_undefined b/infra/cifuzz/test_data/undefined/out/curl_fuzzer_undefined Binary files differindex 504cab108..504cab108 100755 --- a/infra/cifuzz/test_files/undefined/out/curl_fuzzer_undefined +++ b/infra/cifuzz/test_data/undefined/out/curl_fuzzer_undefined diff --git a/infra/go/coverage/gocovsum/gocovsum.go b/infra/go/coverage/gocovsum/gocovsum.go deleted file mode 100644 index 206600619..000000000 --- a/infra/go/coverage/gocovsum/gocovsum.go +++ /dev/null @@ -1,126 +0,0 @@ -package main - -import ( - "encoding/json" - "flag" - "fmt" - "log" - - "go/ast" - "go/parser" - "go/token" - "os" - "path" - - "golang.org/x/tools/cover" -) - -type CoverageTotal struct { - Count int `json:"count"` - Covered int `json:"covered"` - Uncovered int `json:"notcovered"` - Percent float64 `json:"percent"` -} - -type CoverageTotals struct { - Functions CoverageTotal `json:"functions,omitempty"` - Lines CoverageTotal `json:"lines,omitempty"` - Regions CoverageTotal `json:"regions,omitempty"` -} - -type CoverageData struct { - Totals CoverageTotals `json:"totals,omitempty"` -} - -type PositionInterval struct { - start token.Position - end token.Position -} - -type CoverageSummary struct { - Data []CoverageData `json:"data,omitempty"` - Type string `json:"type,omitempty"` - Version string `json:"version,omitempty"` -} - -func isFunctionCovered(s token.Position, e token.Position, blocks []cover.ProfileBlock) bool { - for _, b := range blocks { - if b.StartLine >= s.Line && b.StartLine <= e.Line && b.EndLine >= s.Line && b.EndLine <= e.Line { - if b.Count > 0 { - return true - } - } - } - return false -} - -func main() { - flag.Parse() - - if len(flag.Args()) != 1 { - log.Fatalf("needs exactly one argument") - } - profiles, err := cover.ParseProfiles(flag.Args()[0]) - if err != nil { - log.Fatalf("failed to parse profiles: %v", err) - } - r := CoverageSummary{} - r.Type = "oss-fuzz.go.coverage.json.export" - r.Version = "1.0.0" - r.Data = make([]CoverageData, 1) - gopath := os.Getenv("GOPATH") - if len(gopath) == 0 { - gopath = os.Getenv("HOME") + "/go" - } - for _, p := range profiles { - fset := token.NewFileSet() // positions are relative to fset - f, err := parser.ParseFile(fset, path.Join(gopath, "src", p.FileName), nil, 0) - if err != nil { - panic(err) - } - ast.Inspect(f, func(n ast.Node) bool { - switch x := n.(type) { - case *ast.FuncLit: - startf := fset.Position(x.Pos()) - endf := fset.Position(x.End()) - r.Data[0].Totals.Functions.Count++ - if isFunctionCovered(startf, endf, p.Blocks) { - r.Data[0].Totals.Functions.Covered++ - } else { - r.Data[0].Totals.Functions.Uncovered++ - } - case *ast.FuncDecl: - startf := fset.Position(x.Pos()) - endf := fset.Position(x.End()) - r.Data[0].Totals.Functions.Count++ - if isFunctionCovered(startf, endf, p.Blocks) { - r.Data[0].Totals.Functions.Covered++ - } else { - r.Data[0].Totals.Functions.Uncovered++ - } - } - return true - }) - - for _, b := range p.Blocks { - r.Data[0].Totals.Regions.Count++ - if b.Count > 0 { - r.Data[0].Totals.Regions.Covered++ - } else { - r.Data[0].Totals.Regions.Uncovered++ - } - - r.Data[0].Totals.Lines.Count += b.NumStmt - if b.Count > 0 { - r.Data[0].Totals.Lines.Covered += b.NumStmt - } else { - r.Data[0].Totals.Lines.Uncovered += b.NumStmt - } - } - } - r.Data[0].Totals.Regions.Percent = float64(100*r.Data[0].Totals.Regions.Covered) / float64(r.Data[0].Totals.Regions.Count) - r.Data[0].Totals.Lines.Percent = float64(100*r.Data[0].Totals.Lines.Covered) / float64(r.Data[0].Totals.Lines.Count) - r.Data[0].Totals.Functions.Percent = float64(100*r.Data[0].Totals.Functions.Covered) / float64(r.Data[0].Totals.Functions.Count) - o, _ := json.Marshal(r) - fmt.Printf(string(o)) -} diff --git a/infra/go/coverage/pprof-merge/go.mod b/infra/go/coverage/pprof-merge/go.mod deleted file mode 100644 index 5d5b514ac..000000000 --- a/infra/go/coverage/pprof-merge/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module github.com/rakyll/pprof-merge - -go 1.13 - -require github.com/google/pprof v0.0.0-20190908185732-236ed259b199 diff --git a/infra/helper.py b/infra/helper.py index 8a0a640c1..e24df4ded 100755 --- a/infra/helper.py +++ b/infra/helper.py @@ -22,7 +22,6 @@ from multiprocessing.dummy import Pool as ThreadPool import argparse import datetime import errno -import multiprocessing import os import pipes import re @@ -59,15 +58,61 @@ CORPUS_BACKUP_URL_FORMAT = ( PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)') # Languages from project.yaml that have code coverage support. -LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go'] +LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust'] +# pylint: disable=too-many-lines -def main(): # pylint: disable=too-many-branches,too-many-return-statements,too-many-statements + +def main(): # pylint: disable=too-many-branches,too-many-return-statements """Get subcommand from program arguments and do it.""" os.chdir(OSS_FUZZ_DIR) if not os.path.exists(BUILD_DIR): os.mkdir(BUILD_DIR) + args = parse_args() + + # We have different default values for `sanitizer` depending on the `engine`. + # Some commands do not have `sanitizer` argument, so `hasattr` is necessary. + if hasattr(args, 'sanitizer') and not args.sanitizer: + if args.engine == 'dataflow': + args.sanitizer = 'dataflow' + else: + args.sanitizer = 'address' + + if args.command == 'generate': + return generate(args) + if args.command == 'build_image': + return build_image(args) + if args.command == 'build_fuzzers': + return build_fuzzers(args) + if args.command == 'check_build': + return check_build(args) + if args.command == 'download_corpora': + return download_corpora(args) + if args.command == 'run_fuzzer': + return run_fuzzer(args) + if args.command == 'coverage': + return coverage(args) + if args.command == 'reproduce': + return reproduce(args) + if args.command == 'shell': + return shell(args) + if args.command == 'pull_images': + return pull_images(args) + + return 0 + + +def parse_args(args=None): + """Parses args using argparser and returns parsed args.""" + # Use default argument None for args so that in production, argparse does its + # normal behavior, but unittesting is easier. + parser = get_parser() + return parser.parse_args(args) + + +def get_parser(): # pylint: disable=too-many-statements + """Returns an argparse parser.""" parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers') subparsers = parser.add_subparsers(dest='command') @@ -112,8 +157,9 @@ def main(): # pylint: disable=too-many-branches,too-many-return-statements,too- _add_engine_args( check_build_parser, choices=['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none']) - _add_sanitizer_args(check_build_parser, - choices=['address', 'memory', 'undefined', 'dataflow']) + _add_sanitizer_args( + check_build_parser, + choices=['address', 'memory', 'undefined', 'dataflow', 'thread']) _add_environment_args(check_build_parser) check_build_parser.add_argument('project_name', help='name of the project') check_build_parser.add_argument('fuzzer_name', @@ -189,39 +235,7 @@ def main(): # pylint: disable=too-many-branches,too-many-return-statements,too- _add_environment_args(shell_parser) subparsers.add_parser('pull_images', help='Pull base images.') - - args = parser.parse_args() - - # We have different default values for `sanitizer` depending on the `engine`. - # Some commands do not have `sanitizer` argument, so `hasattr` is necessary. - if hasattr(args, 'sanitizer') and not args.sanitizer: - if args.engine == 'dataflow': - args.sanitizer = 'dataflow' - else: - args.sanitizer = 'address' - - if args.command == 'generate': - return generate(args) - if args.command == 'build_image': - return build_image(args) - if args.command == 'build_fuzzers': - return build_fuzzers(args) - if args.command == 'check_build': - return check_build(args) - if args.command == 'download_corpora': - return download_corpora(args) - if args.command == 'run_fuzzer': - return run_fuzzer(args) - if args.command == 'coverage': - return coverage(args) - if args.command == 'reproduce': - return reproduce(args) - if args.command == 'shell': - return shell(args) - if args.command == 'pull_images': - return pull_images(args) - - return 0 + return parser def is_base_image(image_name): @@ -335,7 +349,7 @@ def _add_engine_args(parser, def _add_sanitizer_args(parser, choices=('address', 'memory', 'undefined', 'coverage', - 'dataflow')): + 'dataflow', 'thread')): """Add common sanitizer args.""" parser.add_argument( '--sanitizer', @@ -632,7 +646,7 @@ def check_build(args): ] if args.fuzzer_name: - run_args += ['test_one', os.path.join('/out', args.fuzzer_name)] + run_args += ['test_one.py', args.fuzzer_name] else: run_args.append('test_all.py') @@ -672,14 +686,14 @@ def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir): fuzz_target=fuzz_target) command = ['gsutil', 'ls', corpus_backup_url] - corpus_listing = subprocess.Popen(command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = corpus_listing.communicate() + # Don't capture stderr. We want it to print in real time, in case gsutil is + # asking for two-factor authentication. + corpus_listing = subprocess.Popen(command, stdout=subprocess.PIPE) + output, _ = corpus_listing.communicate() # Some fuzz targets (e.g. new ones) may not have corpus yet, just skip those. if corpus_listing.returncode: - print('WARNING: corpus for {0} not found:\n{1}'.format(fuzz_target, error), + print('WARNING: corpus for {0} not found:\n'.format(fuzz_target), file=sys.stderr) return @@ -736,7 +750,7 @@ def download_corpora(args): print('Downloading corpora for %s project to %s' % (args.project_name, corpus_dir)) - thread_pool = ThreadPool(multiprocessing.cpu_count()) + thread_pool = ThreadPool() return all(thread_pool.map(_download_for_single_target, fuzz_targets)) @@ -956,9 +970,11 @@ def shell(args): 'FUZZING_ENGINE=' + args.engine, 'SANITIZER=' + args.sanitizer, 'ARCHITECTURE=' + args.architecture, - 'FUZZING_LANGUAGE=' + _get_project_language(args.project_name), ] + if args.project_name != 'base-runner-debug': + env.append('FUZZING_LANGUAGE=' + _get_project_language(args.project_name)) + if args.e: env += args.e diff --git a/infra/helper_test.py b/infra/helper_test.py new file mode 100644 index 000000000..d899a835b --- /dev/null +++ b/infra/helper_test.py @@ -0,0 +1,35 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for helper.py""" + +import unittest +from unittest import mock + +import helper + + +class TestShell(unittest.TestCase): + """Tests 'shell' command.""" + + @mock.patch('helper.docker_run') + @mock.patch('helper.build_image_impl') + def test_base_runner_debug(self, mocked_build_image_impl, _): + """Tests that shell base-runner-debug works as intended.""" + image_name = 'base-runner-debug' + unparsed_args = ['shell', image_name] + args = helper.parse_args(unparsed_args) + args.sanitizer = 'address' + result = helper.shell(args) + mocked_build_image_impl.assert_called_with(image_name) + self.assertEqual(result, 0) diff --git a/infra/presubmit.py b/infra/presubmit.py index bd5e9c327..90b4f90ac 100755 --- a/infra/presubmit.py +++ b/infra/presubmit.py @@ -104,6 +104,7 @@ class ProjectYamlChecker: 'c', 'c++', 'go', + 'jvm', 'python', 'rust', ] @@ -381,8 +382,9 @@ def run_nonbuild_tests(parallel): def run_tests(_=None, parallel=False): """Runs all unit tests.""" - success = run_nonbuild_tests(parallel) - return success and run_build_tests() + nonbuild_success = run_nonbuild_tests(parallel) + build_success = run_build_tests() + return nonbuild_success and build_success def get_all_files(): diff --git a/infra/repo_manager.py b/infra/repo_manager.py index a5781b89a..a0b97b3ef 100644 --- a/infra/repo_manager.py +++ b/infra/repo_manager.py @@ -127,6 +127,14 @@ class RepoManager: return out.strip() + def fetch_all_remotes(self): + """Fetch all remotes for checkouts that track a single branch.""" + self.git([ + 'config', 'remote.origin.fetch', '+refs/heads/*:refs/remotes/origin/*' + ], + check_result=True) + self.git(['remote', 'update'], check_result=True) + def get_commit_list(self, newest_commit, oldest_commit=None): """Gets the list of commits(inclusive) between the old and new commits. diff --git a/infra/testcases/curl_test_data b/infra/testcases/curl_test_data Binary files differdeleted file mode 100644 index ed4b54ea3..000000000 --- a/infra/testcases/curl_test_data +++ /dev/null diff --git a/infra/testcases/libarchive_test_data b/infra/testcases/libarchive_test_data Binary files differdeleted file mode 100644 index 928bfec97..000000000 --- a/infra/testcases/libarchive_test_data +++ /dev/null diff --git a/infra/testcases/ndpi_test_data b/infra/testcases/ndpi_test_data Binary files differdeleted file mode 100644 index 010af8604..000000000 --- a/infra/testcases/ndpi_test_data +++ /dev/null diff --git a/infra/testcases/usrsctp_test_data b/infra/testcases/usrsctp_test_data Binary files differdeleted file mode 100644 index fa90322a2..000000000 --- a/infra/testcases/usrsctp_test_data +++ /dev/null diff --git a/infra/testcases/yara_test_data b/infra/testcases/yara_test_data deleted file mode 100644 index e2a0b94af..000000000 --- a/infra/testcases/yara_test_data +++ /dev/null @@ -1 +0,0 @@ -rule N{condition:for 1r in r(r
\ No newline at end of file diff --git a/infra/utils_test.py b/infra/utils_test.py index a56295c93..aa6ec7ba7 100644 --- a/infra/utils_test.py +++ b/infra/utils_test.py @@ -24,7 +24,7 @@ import helper EXAMPLE_PROJECT = 'example' TEST_OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'cifuzz', 'test_files', 'out') + 'cifuzz', 'test_data', 'out') class IsFuzzTargetLocalTest(unittest.TestCase): |