aboutsummaryrefslogtreecommitdiff
path: root/infra
diff options
context:
space:
mode:
authorElliott Hughes <enh@google.com>2021-04-02 19:51:01 +0000
committerAutomerger Merge Worker <android-build-automerger-merge-worker@system.gserviceaccount.com>2021-04-02 19:51:01 +0000
commit75c8dcf71ca8652f671b4ca5fea780a558c86e08 (patch)
tree328e6c9629b196cec1de3a94ee804d9fee3a0524 /infra
parent378a8d19d33a5a62afbbe33b7f7b87b67db47236 (diff)
parent235e96b2f8ab4e43316158a2e6fa69e75a219e23 (diff)
downloadoss-fuzz-75c8dcf71ca8652f671b4ca5fea780a558c86e08.tar.gz
Original change: https://android-review.googlesource.com/c/platform/external/oss-fuzz/+/1662261 Change-Id: Ib7b7a79b38e1261c1d5fb4ccb1a5dfd106588996
Diffstat (limited to 'infra')
-rw-r--r--infra/.dockerignore10
-rw-r--r--infra/.pylintrc2
-rw-r--r--infra/base-images/Jenkinsfile37
-rw-r--r--infra/base-images/base-builder/Dockerfile51
-rwxr-xr-xinfra/base-images/base-builder/bazel_build_fuzz_tests80
-rwxr-xr-xinfra/base-images/base-builder/cargo51
-rwxr-xr-xinfra/base-images/base-builder/compile30
-rw-r--r--infra/base-images/base-builder/compile_afl83
-rwxr-xr-xinfra/base-images/base-builder/compile_go_fuzzer29
-rwxr-xr-xinfra/base-images/base-builder/compile_honggfuzz4
-rwxr-xr-xinfra/base-images/base-builder/compile_libfuzzer13
-rwxr-xr-xinfra/base-images/base-builder/debug_afl40
-rw-r--r--infra/base-images/base-builder/detect_repo.py21
-rw-r--r--infra/base-images/base-builder/detect_repo_test.py28
-rwxr-xr-xinfra/base-images/base-builder/precompile_afl35
-rwxr-xr-xinfra/base-images/base-builder/precompile_honggfuzz7
-rw-r--r--infra/base-images/base-clang/Dockerfile3
-rwxr-xr-xinfra/base-images/base-clang/checkout_build_install_llvm.sh72
-rwxr-xr-x[-rw-r--r--]infra/base-images/base-runner/Dockerfile38
-rwxr-xr-xinfra/base-images/base-runner/bad_build_check18
-rwxr-xr-xinfra/base-images/base-runner/coverage12
-rw-r--r--infra/base-images/base-runner/gocoverage/go.mod8
-rw-r--r--infra/base-images/base-runner/gocoverage/go.sum30
-rw-r--r--infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE (renamed from infra/go/coverage/gocovmerge/LICENSE)0
-rw-r--r--infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go (renamed from infra/go/coverage/gocovmerge/gocovmerge.go)0
-rw-r--r--infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go147
-rw-r--r--infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE (renamed from infra/go/coverage/pprof-merge/LICENSE)0
-rw-r--r--infra/base-images/base-runner/gocoverage/pprof-merge/main.go (renamed from infra/go/coverage/pprof-merge/main.go)0
-rwxr-xr-xinfra/base-images/base-runner/minijail0bin59392 -> 0 bytes
-rwxr-xr-xinfra/base-images/base-runner/rcfilt21
-rwxr-xr-xinfra/base-images/base-runner/run_fuzzer13
-rwxr-xr-xinfra/base-images/base-runner/run_minijail59
-rwxr-xr-xinfra/base-images/base-runner/test_all.py7
-rwxr-xr-xinfra/base-images/base-runner/test_one58
-rwxr-xr-xinfra/base-images/base-runner/test_one.py43
-rwxr-xr-xinfra/base-images/base-sanitizer-libs-builder/msan_build.py80
-rw-r--r--infra/bisector.py2
-rw-r--r--infra/build/functions/build_and_run_coverage.py2
-rw-r--r--infra/build/functions/requirements.txt2
-rw-r--r--infra/build/functions/update_build_status.py15
-rw-r--r--infra/build_specified_commit.py1
-rw-r--r--infra/build_specified_commit_test.py4
-rwxr-xr-xinfra/ci/build.py2
-rw-r--r--infra/ci/requirements.txt2
-rw-r--r--infra/cifuzz/actions/build_fuzzers/action.yml11
-rw-r--r--infra/cifuzz/actions/run_fuzzers/action.yml6
-rw-r--r--infra/cifuzz/affected_fuzz_targets_test.py8
-rw-r--r--infra/cifuzz/build_fuzzers.py24
-rw-r--r--infra/cifuzz/build_fuzzers_entrypoint.py10
-rw-r--r--infra/cifuzz/build_fuzzers_test.py27
-rw-r--r--infra/cifuzz/cifuzz-base/Dockerfile24
-rw-r--r--infra/cifuzz/config_utils.py32
-rw-r--r--infra/cifuzz/config_utils_test.py29
-rw-r--r--infra/cifuzz/coverage.py2
-rw-r--r--infra/cifuzz/coverage_test.py52
-rw-r--r--infra/cifuzz/docker.py38
-rw-r--r--infra/cifuzz/environment.py54
-rw-r--r--infra/cifuzz/fuzz_target.py46
-rw-r--r--infra/cifuzz/fuzz_target_test.py3
-rw-r--r--infra/cifuzz/run_fuzzers.py20
-rw-r--r--infra/cifuzz/run_fuzzers_entrypoint.py19
-rw-r--r--infra/cifuzz/run_fuzzers_test.py32
-rw-r--r--infra/cifuzz/stack_parser.py22
-rw-r--r--infra/cifuzz/stack_parser_test.py33
-rw-r--r--infra/cifuzz/test_data/example_crash_fuzzer_bug_summary.txt (renamed from infra/cifuzz/test_files/bug_summary_example.txt)0
-rw-r--r--infra/cifuzz/test_data/example_crash_fuzzer_output.txt (renamed from infra/cifuzz/test_files/example_crash_fuzzer_output.txt)0
-rw-r--r--infra/cifuzz/test_data/example_curl_cov.json (renamed from infra/cifuzz/test_files/example_curl_cov.json)0
-rw-r--r--infra/cifuzz/test_data/example_curl_file_list.json (renamed from infra/cifuzz/test_files/example_curl_file_list.json)0
-rw-r--r--infra/cifuzz/test_data/example_curl_fuzzer_cov.json (renamed from infra/cifuzz/test_files/example_curl_fuzzer_cov.json)0
-rw-r--r--infra/cifuzz/test_data/external-project/Makefile (renamed from infra/cifuzz/test_files/external-project/Makefile)0
-rw-r--r--infra/cifuzz/test_data/external-project/do_stuff_fuzzer.cpp (renamed from infra/cifuzz/test_files/external-project/do_stuff_fuzzer.cpp)0
-rw-r--r--infra/cifuzz/test_data/external-project/do_stuff_fuzzer.dict (renamed from infra/cifuzz/test_files/external-project/do_stuff_fuzzer.dict)0
-rw-r--r--infra/cifuzz/test_data/external-project/my_api.cpp (renamed from infra/cifuzz/test_files/external-project/my_api.cpp)0
-rw-r--r--infra/cifuzz/test_data/external-project/my_api.h (renamed from infra/cifuzz/test_files/external-project/my_api.h)0
-rw-r--r--infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile (renamed from infra/cifuzz/test_files/external-project/oss-fuzz/Dockerfile)0
-rw-r--r--infra/cifuzz/test_data/external-project/oss-fuzz/build.sh (renamed from infra/cifuzz/test_files/external-project/oss-fuzz/build.sh)0
-rw-r--r--infra/cifuzz/test_data/external-project/standalone_fuzz_target_runner.cpp (renamed from infra/cifuzz/test_files/external-project/standalone_fuzz_target_runner.cpp)0
-rwxr-xr-xinfra/cifuzz/test_data/memory/out/curl_fuzzer_memory (renamed from infra/cifuzz/test_files/memory/out/curl_fuzzer_memory)bin9768680 -> 9768680 bytes
-rw-r--r--infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt22
-rw-r--r--infra/cifuzz/test_data/msan_crash_fuzzer_output.txt39
-rwxr-xr-xinfra/cifuzz/test_data/out/example_crash_fuzzer (renamed from infra/cifuzz/test_files/out/example_crash_fuzzer)bin4375872 -> 4375872 bytes
-rwxr-xr-xinfra/cifuzz/test_data/out/example_nocrash_fuzzer (renamed from infra/cifuzz/test_files/out/example_nocrash_fuzzer)bin4376224 -> 4376224 bytes
-rwxr-xr-xinfra/cifuzz/test_data/undefined/out/curl_fuzzer_undefined (renamed from infra/cifuzz/test_files/undefined/out/curl_fuzzer_undefined)bin14401312 -> 14401312 bytes
-rw-r--r--infra/go/coverage/gocovsum/gocovsum.go126
-rw-r--r--infra/go/coverage/pprof-merge/go.mod5
-rwxr-xr-xinfra/helper.py110
-rw-r--r--infra/helper_test.py35
-rwxr-xr-xinfra/presubmit.py6
-rw-r--r--infra/repo_manager.py8
-rw-r--r--infra/testcases/curl_test_databin456 -> 0 bytes
-rw-r--r--infra/testcases/libarchive_test_databin156184 -> 0 bytes
-rw-r--r--infra/testcases/ndpi_test_databin3134 -> 0 bytes
-rw-r--r--infra/testcases/usrsctp_test_databin73 -> 0 bytes
-rw-r--r--infra/testcases/yara_test_data1
-rw-r--r--infra/utils_test.py2
95 files changed, 1411 insertions, 595 deletions
diff --git a/infra/.dockerignore b/infra/.dockerignore
index 1e82e3e75..c78653342 100644
--- a/infra/.dockerignore
+++ b/infra/.dockerignore
@@ -1 +1,9 @@
-test_files \ No newline at end of file
+cifuzz/test_data/*
+
+# Copied from .gitignore.
+.vscode/
+*.pyc
+build
+*~
+.DS_Store
+*.swp \ No newline at end of file
diff --git a/infra/.pylintrc b/infra/.pylintrc
deleted file mode 100644
index 8ce0b2226..000000000
--- a/infra/.pylintrc
+++ /dev/null
@@ -1,2 +0,0 @@
-[FORMAT]
-indent-string = " " \ No newline at end of file
diff --git a/infra/base-images/Jenkinsfile b/infra/base-images/Jenkinsfile
deleted file mode 100644
index 67b22e9b0..000000000
--- a/infra/base-images/Jenkinsfile
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2016 Google Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-////////////////////////////////////////////////////////////////////////////////
-
-// Jenkins build script for base images.
-node {
- git url: 'https://github.com/google/oss-fuzz/'
-
- stage("infra/base-images/all.sh") {
- sh "infra/base-images/all.sh --no-cache"
- }
-
- stage("docker push") {
- def images = ['ossfuzz/base-image', 'ossfuzz/base-clang', 'ossfuzz/base-libfuzzer',
- 'ossfuzz/base-runner', 'ossfuzz/base-runner-debug',
- 'ossfuzz/base-builder',]
-
- docker.withRegistry('', 'docker-login') {
- for (int i = 0; i < images.size(); i++) {
- def image = images[i]
- docker.image(image).push()
- }
- }
- }
-}
diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile
index 6f596d5ba..d802f247a 100644
--- a/infra/base-images/base-builder/Dockerfile
+++ b/infra/base-images/base-builder/Dockerfile
@@ -29,6 +29,7 @@ RUN dpkg --add-architecture i386 && \
jq \
libc6-dev-i386 \
patchelf \
+ rsync \
subversion \
zip
@@ -57,12 +58,14 @@ RUN export PYTHON_DEPS="\
ln -s /usr/bin/python3 /usr/bin/python && \
cd .. && \
rm -r /tmp/Python-$PYTHON_VERSION.tar.xz /tmp/Python-$PYTHON_VERSION && \
- apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888
+ rm -rf /usr/local/lib/python3.8/test && \
+ apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888
# Install latest atheris for python fuzzing, pyinstaller for fuzzer packaging,
# six for Bazel rules.
-RUN unset CFLAGS CXXFLAGS && pip3 install -v \
- atheris pyinstaller==4.1 six==1.15.0
+RUN unset CFLAGS CXXFLAGS && pip3 install -v --no-cache-dir \
+ atheris pyinstaller==4.1 six==1.15.0 && \
+ rm -rf /tmp/*
# Download and install the latest stable Go.
RUN cd /tmp && \
@@ -87,15 +90,42 @@ ENV CARGO_HOME=/rust
ENV RUSTUP_HOME=/rust/rustup
ENV PATH=$PATH:/rust/bin
RUN curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal
-RUN cargo install cargo-fuzz
+RUN cargo install cargo-fuzz && rm -rf /rust/registry
# Needed to recompile rust std library for MSAN
RUN rustup component add rust-src --toolchain nightly
+# Set up custom environment variable for source code copy for coverage reports
+ENV OSSFUZZ_RUSTPATH /rust
# Install Bazel through Bazelisk, which automatically fetches the latest Bazel version.
ENV BAZELISK_VERSION 1.7.4
RUN curl -L https://github.com/bazelbuild/bazelisk/releases/download/v$BAZELISK_VERSION/bazelisk-linux-amd64 -o /usr/local/bin/bazel && \
chmod +x /usr/local/bin/bazel
+# Install OpenJDK 15 and trim its size by removing unused components.
+ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64
+ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server
+ENV PATH=$PATH:$JAVA_HOME/bin
+RUN cd /tmp && \
+ curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \
+ mkdir -p $JAVA_HOME && \
+ tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \
+ rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \
+ rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip
+
+# Install the latest Jazzer in $OUT.
+# jazzer_api_deploy.jar is required only at build-time, the agent and the
+# drivers are copied to $OUT as they need to be present on the runners.
+ENV JAZZER_API_PATH "/usr/local/lib/jazzer_api_deploy.jar"
+RUN cd $SRC/ && \
+ git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \
+ cd jazzer && \
+ bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \
+ //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //agent:jazzer_api_deploy.jar && \
+ cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan /usr/local/bin/ && \
+ cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH && \
+ rm -rf ~/.cache/bazel ~/.cache/bazelisk && \
+ rm -rf $SRC/jazzer
+
# Default build flags for various sanitizers.
ENV SANITIZER_FLAGS_address "-fsanitize=address -fsanitize-address-use-after-scope"
@@ -106,6 +136,8 @@ ENV SANITIZER_FLAGS_memory "-fsanitize=memory -fsanitize-memory-track-origins"
ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow"
+ENV SANITIZER_FLAGS_thread "-fsanitize=thread"
+
# Do not use any sanitizers in the coverage build.
ENV SANITIZER_FLAGS_coverage ""
@@ -144,15 +176,12 @@ ENV LIB_FUZZING_ENGINE="/usr/lib/libFuzzingEngine.a"
# TODO: remove after tpm2 catchup.
ENV FUZZER_LDFLAGS ""
-ENV PRECOMPILED_DIR="/usr/lib/precompiled"
-RUN mkdir $PRECOMPILED_DIR
-
WORKDIR $SRC
# TODO: switch to -b stable once we can.
RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \
cd aflplusplus && \
- git checkout aeb7d7048371cd91ab9280c3958f1c35e5d5e758
+ git checkout 2102264acf5c271b7560a82771b3af8136af9354
RUN cd $SRC && \
curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \
@@ -161,12 +190,14 @@ RUN cd $SRC && \
tar -xzv --strip-components=1 -f $SRC/oss-fuzz.tar.gz && \
rm -rf examples $SRC/oss-fuzz.tar.gz
-COPY compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
- compile_go_fuzzer precompile_honggfuzz srcmap write_labels.py /usr/local/bin/
+COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
+ compile_go_fuzzer precompile_honggfuzz precompile_afl debug_afl srcmap \
+ write_labels.py bazel_build_fuzz_tests /usr/local/bin/
COPY detect_repo.py /opt/cifuzz/
COPY ossfuzz_coverage_runner.go $GOPATH
RUN precompile_honggfuzz
+RUN precompile_afl
CMD ["compile"]
diff --git a/infra/base-images/base-builder/bazel_build_fuzz_tests b/infra/base-images/base-builder/bazel_build_fuzz_tests
new file mode 100755
index 000000000..86740ee01
--- /dev/null
+++ b/infra/base-images/base-builder/bazel_build_fuzz_tests
@@ -0,0 +1,80 @@
+#!/bin/bash -eu
+#
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+: "${BAZEL_FUZZ_TEST_TAG:=fuzz-test}"
+: "${BAZEL_FUZZ_TEST_EXCLUDE_TAG:=no-oss-fuzz}"
+: "${BAZEL_PACKAGE_SUFFIX:=_oss_fuzz}"
+: "${BAZEL_TOOL:=bazel}"
+: "${BAZEL_EXTRA_BUILD_FLAGS:=}"
+
+if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then
+ BAZEL_FUZZ_TEST_QUERY="
+ let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in
+ \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests)
+ "
+fi
+
+echo "Using Bazel query to find fuzz targets: ${BAZEL_FUZZ_TEST_QUERY}"
+
+declare -r OSS_FUZZ_TESTS=(
+ $(bazel query "${BAZEL_FUZZ_TEST_QUERY}" | sed "s/$/${BAZEL_PACKAGE_SUFFIX}/")
+)
+
+echo "Found ${#OSS_FUZZ_TESTS[@]} fuzz test packages:"
+for oss_fuzz_test in "${OSS_FUZZ_TESTS[@]}"; do
+ echo " ${oss_fuzz_test}"
+done
+
+declare -r BAZEL_BUILD_FLAGS=(
+ "-c" "opt"
+ "--//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \
+ "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \
+ "--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \
+ "--linkopt=-lc++" \
+ "--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \
+ ${BAZEL_EXTRA_BUILD_FLAGS[*]}
+)
+
+echo "Building the fuzz tests with the following Bazel options:"
+echo " ${BAZEL_BUILD_FLAGS[@]}"
+
+${BAZEL_TOOL} build "${BAZEL_BUILD_FLAGS[@]}" "${OSS_FUZZ_TESTS[@]}"
+
+echo "Extracting the fuzz test packages in the output directory."
+for oss_fuzz_archive in $(find bazel-bin/ -name "*${BAZEL_PACKAGE_SUFFIX}.tar"); do
+ tar -xvf "${oss_fuzz_archive}" -C "${OUT}"
+done
+
+if [ "$SANITIZER" = "coverage" ]; then
+ echo "Collecting the repository source files for coverage tracking."
+ declare -r COVERAGE_SOURCES="${OUT}/proc/self/cwd"
+ mkdir -p "${COVERAGE_SOURCES}"
+ declare -r RSYNC_FILTER_ARGS=(
+ "--include" "*.h"
+ "--include" "*.cc"
+ "--include" "*.hpp"
+ "--include" "*.cpp"
+ "--include" "*.c"
+ "--include" "*.inc"
+ "--include" "*/"
+ "--exclude" "*"
+ )
+ rsync -avLk "${RSYNC_FILTER_ARGS[@]}" \
+ "$(bazel info execution_root)/" \
+ "${COVERAGE_SOURCES}/"
+fi
diff --git a/infra/base-images/base-builder/cargo b/infra/base-images/base-builder/cargo
new file mode 100755
index 000000000..bed8e7660
--- /dev/null
+++ b/infra/base-images/base-builder/cargo
@@ -0,0 +1,51 @@
+#!/bin/bash -eu
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This is a wrapper around calling cargo
+# This just expands RUSTFLAGS in case of a coverage build
+# We need this until https://github.com/rust-lang/cargo/issues/5450 is merged
+# because cargo uses relative paths for the current crate
+# and absolute paths for its dependencies
+#
+################################################################################
+
+if [ "$SANITIZER" = "coverage" ] && [ $1 = "build" ]
+then
+ crate_src_abspath=`cargo metadata --no-deps --format-version 1 | jq -r '.workspace_root'`
+ export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix src=$crate_src_abspath/src"
+fi
+
+if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ]
+then
+ # hack to turn cargo fuzz build into cargo build so as to get coverage
+ # cargo fuzz adds "--target" "x86_64-unknown-linux-gnu"
+ (
+ # go into fuzz directory if not already the case
+ cd fuzz || true
+ fuzz_src_abspath=`pwd`
+ export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix fuzz_targets=$fuzz_src_abspath/fuzz_targets"
+ # we do not want to trigger debug assertions and stops
+ export RUSTFLAGS="$RUSTFLAGS -C debug-assertions=no"
+ # do not optimize with --release, leading to Malformed instrumentation profile data
+ cargo build --bins
+ # copies the build output in the expected target directory
+ cd `cargo metadata --format-version 1 --no-deps | jq -r '.target_directory'`
+ mkdir -p x86_64-unknown-linux-gnu/release
+ cp -r debug/* x86_64-unknown-linux-gnu/release/
+ )
+ exit 0
+fi
+
+/rust/bin/cargo "$@"
diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile
index 2bf20b1e3..78453c98c 100755
--- a/infra/base-images/base-builder/compile
+++ b/infra/base-images/base-builder/compile
@@ -22,6 +22,21 @@ if [ "$SANITIZER" = "dataflow" ] && [ "$FUZZING_ENGINE" != "dataflow" ]; then
exit 1
fi
+if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+ if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then
+ echo "ERROR: JVM projects can be fuzzed with libFuzzer engine only."
+ exit 1
+ fi
+ if [ "$SANITIZER" != "address" ]; then
+ echo "ERROR: JVM projects can be fuzzed with AddressSanitizer only."
+ exit 1
+ fi
+ if [ "$ARCHITECTURE" != "x86_64" ]; then
+ echo "ERROR: JVM projects can be fuzzed on x86_64 architecture only."
+ exit 1
+ fi
+fi
+
if [ "$FUZZING_LANGUAGE" = "python" ]; then
if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then
echo "ERROR: Python projects can be fuzzed with libFuzzer engine only."
@@ -46,7 +61,8 @@ if [[ $ARCHITECTURE == "i386" ]]; then
export CFLAGS="-m32 $CFLAGS"
cp -R /usr/i386/lib/* /usr/lib
fi
-if [[ $FUZZING_ENGINE != "none" ]]; then
+# JVM projects are fuzzed with Jazzer, which has libFuzzer built in.
+if [[ $FUZZING_ENGINE != "none" ]] && [[ $FUZZING_LANGUAGE != "jvm" ]]; then
# compile script might override environment, use . to call it.
. compile_${FUZZING_ENGINE}
fi
@@ -87,6 +103,11 @@ if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$ARCH
else
export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers"
fi
+if [ "$SANITIZER" = "coverage" ]
+then
+ # link to C++ from comment in f5098035eb1a14aa966c8651d88ea3d64323823d
+ export RUSTFLAGS="$RUSTFLAGS -Zinstrument-coverage -C link-arg=-lc++"
+fi
# Add Rust libfuzzer flags.
# See https://github.com/rust-fuzz/libfuzzer/blob/master/build.rs#L12.
@@ -113,6 +134,11 @@ fi
# Copy latest llvm-symbolizer in $OUT for stack symbolization.
cp $(which llvm-symbolizer) $OUT/
+# Copy Jazzer to $OUT if needed.
+if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+ cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $(which jazzer_driver_asan) $OUT/
+fi
+
echo "---------------------------------------------------------------"
echo "CC=$CC"
echo "CXX=$CXX"
@@ -124,7 +150,7 @@ BUILD_CMD="bash -eux $SRC/build.sh"
# We need to preserve source code files for generating a code coverage report.
# We need exact files that were compiled, so copy both $SRC and $WORK dirs.
-COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OUT"
+COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH $OUT"
if [ "${BUILD_UID-0}" -ne "0" ]; then
adduser -u $BUILD_UID --disabled-password --gecos '' builder
diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl
index 318eca44e..dc6624459 100644
--- a/infra/base-images/base-builder/compile_afl
+++ b/infra/base-images/base-builder/compile_afl
@@ -15,43 +15,78 @@
#
################################################################################
-echo "Compiling afl++"
+# afl++ configuration options.
+# The 'env|grep' setup ensures we do not trigger the linter.
+# The variables need to be set to "1" here - or before running this script.
-# Build and copy afl++ tools necessary for fuzzing.
+# AFL++ settings.
+export AFL_LLVM_MODE_WORKAROUND=0
+export AFL_ENABLE_DICTIONARY=0
+
+# Start compiling afl++.
+echo "Copying precompiled afl++"
+
+# Copy afl++ tools necessary for fuzzing.
pushd $SRC/aflplusplus > /dev/null
-# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it
-# down with sanitizers.
-INITIAL_CXXFLAGS=$CXXFLAGS
-INITIAL_CFLAGS=$CFLAGS
-unset CXXFLAGS
-unset CFLAGS
-make clean
-AFL_NO_X86=1 PYTHON_INCLUDE=/ make
-CFLAGS=$INITIAL_CFLAGS
-CXXFLAGS=$INITIAL_CXXFLAGS
-
-# Build afl++ driver with existing CFLAGS, CXXFLAGS.
-make -C utils/aflpp_driver
-cp libAFLDriver.a $LIB_FUZZING_ENGINE
+cp -f libAFLDriver.a $LIB_FUZZING_ENGINE
# Some important projects include libraries, copy those even when they don't
# start with "afl-". Use "sort -u" to avoid a warning about duplicates.
ls afl-* *.txt *.a *.o *.so | sort -u | xargs cp -t $OUT
-popd > /dev/null
-
export CC="$SRC/aflplusplus/afl-clang-fast"
export CXX="$SRC/aflplusplus/afl-clang-fast++"
# Set sane afl++ environment defaults:
# Be quiet, otherwise this can break some builds.
export AFL_QUIET=1
-# Several targets run their own tools, so ensure its working.
-export AFL_MAP_SIZE=4194304
# No leak errors during builds.
-export ASAN_OPTIONS="detect_leaks=0:symbolize=0"
-#
-# Placeholder for the upcoming afl++ build options roulette
-#
+export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1"
+
+# AFL compile option roulette. It is OK if they all happen together.
+
+# 40% chance to perform CMPLOG
+rm -f "$OUT/afl_cmplog.txt"
+test $(($RANDOM % 10)) -lt 4 && {
+ export AFL_LLVM_CMPLOG=1
+ touch "$OUT/afl_cmplog.txt"
+}
+
+# 10% chance to perform LAF_INTEL
+test $(($RANDOM % 10)) -lt 1 && {
+ export AFL_LLVM_LAF_ALL=1
+}
+
+# If the targets wants a dictionary - then create one.
+test "$AFL_ENABLE_DICTIONARY" = "1" && {
+ export AFL_LLVM_DICT2FILE="$OUT/afl++.dict"
+}
+
+# In case afl-clang-fast ever breaks, this is a workaround:
+test "$AFL_LLVM_MODE_WORKAROUND" = "1" && {
+ export CC=clang
+ export CXX=clang++
+ WORKAROUND_FLAGS=-fsanitize-coverage=trace-pc-guard
+ # We can still do CMPLOG light:
+ test -e "$OUT/afl_cmplog.txt" && {
+ WORKAROUND_FLAGS="$WORKAROUND_FLAGS",trace-cmp
+ }
+ export CFLAGS="$CFLAGS $WORKAROUND_FLAGS"
+ export CXXFLAGS="$CXXFLAGS $WORKAROUND_FLAGS"
+ unset AFL_LLVM_LAF_ALL
+ unset AFL_LLVM_DICT2FILE
+ unset AFL_ENABLE_DICTIONARY
+ # We need to create a new fuzzer lib however.
+ ar ru libAFLDrivernew.a afl-compiler-rt.o utils/aflpp_driver/aflpp_driver.o
+ cp -f libAFLDrivernew.a $LIB_FUZZING_ENGINE
+}
+
+# Provide a way to document the afl++ options used in this build:
+echo
+echo afl++ target compilation setup:
+env | grep AFL_ | tee "$OUT/afl_options.txt"
+echo
+
+popd > /dev/null
echo " done."
diff --git a/infra/base-images/base-builder/compile_go_fuzzer b/infra/base-images/base-builder/compile_go_fuzzer
index 8f8cde759..2342800fb 100755
--- a/infra/base-images/base-builder/compile_go_fuzzer
+++ b/infra/base-images/base-builder/compile_go_fuzzer
@@ -18,27 +18,35 @@
path=$1
function=$2
fuzzer=$3
-tags=""
+tags="-tags gofuzz"
if [[ $# -eq 4 ]]; then
tags="-tags $4"
fi
+# makes directory change temporary
+(
+cd $GOPATH/src/$path || true
+# in the case we are in the right directory, with go.mod but no go.sum
+go mod tidy || true
+# project was downloaded with go get if go list fails
+go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; }
+# project does not have go.mod if go list fails again
+go list $tags $path || { go mod init $path && go mod tidy ;}
+
if [[ $SANITIZER = *coverage* ]]; then
- cd $GOPATH/src/$path
- fuzzed_package=`go list $tags -f '{{.Name}}'`
+ fuzzed_package=`go list $tags -f '{{.Name}}' $path`
+ abspath=`go list $tags -f {{.Dir}} $path`
+ cd $abspath
cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go
sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go
sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
- echo "#!/bin/sh" > $OUT/$fuzzer
- echo "cd $path" >> $OUT/$fuzzer
- # The fuzzer may be in a subdirectory, but we want the coverage report for the whole repository
fuzzed_repo=`echo $path | cut -d/ -f-3`
- echo "go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -coverprofile \$1 " >> $OUT/$fuzzer
- chmod +x $OUT/$fuzzer
-
- cd -
+ abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo`
+ # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir
+ echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath
+ go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path
else
# Compile and instrument all Go files relevant to this fuzz target.
echo "Running go-fuzz $tags -func $function -o $fuzzer.a $path"
@@ -47,3 +55,4 @@ else
# Link Go code ($fuzzer.a) with fuzzing engine to produce fuzz target binary.
$CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer
fi
+)
diff --git a/infra/base-images/base-builder/compile_honggfuzz b/infra/base-images/base-builder/compile_honggfuzz
index 362a0a598..f86e8426d 100755
--- a/infra/base-images/base-builder/compile_honggfuzz
+++ b/infra/base-images/base-builder/compile_honggfuzz
@@ -17,8 +17,8 @@
echo "Skipping compilation; using precompiled honggfuzz"
-cp $PRECOMPILED_DIR/honggfuzz.a $LIB_FUZZING_ENGINE
-cp $PRECOMPILED_DIR/honggfuzz $OUT/
+cp $SRC/honggfuzz/honggfuzz.a $LIB_FUZZING_ENGINE
+cp $SRC/honggfuzz/honggfuzz $OUT/
# Custom coverage flags, roughly in sync with:
# https://github.com/google/honggfuzz/blob/oss-fuzz/hfuzz_cc/hfuzz-cc.c
diff --git a/infra/base-images/base-builder/compile_libfuzzer b/infra/base-images/base-builder/compile_libfuzzer
index 00f2d6337..3fd7f3906 100755
--- a/infra/base-images/base-builder/compile_libfuzzer
+++ b/infra/base-images/base-builder/compile_libfuzzer
@@ -16,16 +16,7 @@
################################################################################
echo -n "Compiling libFuzzer to $LIB_FUZZING_ENGINE... "
-mkdir -p $WORK/libfuzzer
-pushd $WORK/libfuzzer > /dev/null
-
-# Use -fPIC to allow preloading (LD_PRELOAD).
-$CXX $CXXFLAGS -std=c++11 -O2 -fPIC $SANITIZER_FLAGS -fno-sanitize=vptr \
- -c $SRC/libfuzzer/*.cpp -I$SRC/libfuzzer
-ar r $LIB_FUZZING_ENGINE_DEPRECATED $WORK/libfuzzer/*.o
-popd > /dev/null
-rm -rf $WORK/libfuzzer
-# Override variable as libFuzzer builds do not link directly against an
-# engine library, but use -fsanitize=fuzzer to instruct clang to do so.
export LIB_FUZZING_ENGINE="-fsanitize=fuzzer"
+cp /usr/local/lib/clang/*/lib/linux/libclang_rt.fuzzer-$ARCHITECTURE.a \
+ $LIB_FUZZING_ENGINE_DEPRECATED
echo " done."
diff --git a/infra/base-images/base-builder/debug_afl b/infra/base-images/base-builder/debug_afl
new file mode 100755
index 000000000..c53dae815
--- /dev/null
+++ b/infra/base-images/base-builder/debug_afl
@@ -0,0 +1,40 @@
+#!/bin/bash
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Source this file for afl++ debug sessions.
+apt-get update
+apt-get install -y strace gdb vim joe psmisc
+
+pushd $SRC/aflplusplus > /dev/null
+git checkout dev
+git pull
+test -n "$1" && { git checkout "$1" ; git pull ; }
+CFLAGS_SAVE="$CFLAGS"
+CXXFLAGS_SAVE="$CXXFLAGS"
+unset CFLAGS
+unset CXXFLAGS
+make
+export CFLAGS="$CFLAGS_SAVE"
+export CXXFLAGS="$CXXFLAGS_SAVE"
+popd > /dev/null
+
+export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1"
+export AFL_LLVM_LAF_ALL=1
+export AFL_LLVM_CMPLOG=1
+touch "$OUT/afl_cmplog.txt"
+export AFL_LLVM_DICT2FILE=$OUT/afl++.dict
+ulimit -c unlimited
diff --git a/infra/base-images/base-builder/detect_repo.py b/infra/base-images/base-builder/detect_repo.py
index 8969e974f..e677e1023 100644
--- a/infra/base-images/base-builder/detect_repo.py
+++ b/infra/base-images/base-builder/detect_repo.py
@@ -107,20 +107,25 @@ def get_repo(repo_path):
return None
-def check_for_repo_name(repo_path, repo_name):
- """Check to see if the repo_name matches the remote repository repo name.
+def check_for_repo_name(repo_path, expected_repo_name):
+ """Returns True if the repo at |repo_path| repo_name matches
+ |expected_repo_name|.
Args:
- repo_path: The directory of the git repo.
- repo_name: The name of the target git repo.
+ repo_path: The directory of a git repo.
+ expected_repo_name: The name of the target git repo.
"""
if not os.path.exists(os.path.join(repo_path, '.git')):
return False
- out, _ = execute(['git', 'config', '--get', 'remote.origin.url'],
- location=repo_path)
- out = out.split('/')[-1].replace('.git', '').rstrip()
- return out == repo_name
+ repo_url, _ = execute(['git', 'config', '--get', 'remote.origin.url'],
+ location=repo_path)
+ # Handle two common cases:
+ # https://github.com/google/syzkaller/
+ # https://github.com/google/syzkaller.git
+ repo_url = repo_url.replace('.git', '').rstrip().rstrip('/')
+ actual_repo_name = repo_url.split('/')[-1]
+ return actual_repo_name == expected_repo_name
def check_for_commit(repo_path, commit):
diff --git a/infra/base-images/base-builder/detect_repo_test.py b/infra/base-images/base-builder/detect_repo_test.py
index 21f64af44..0243b3ac5 100644
--- a/infra/base-images/base-builder/detect_repo_test.py
+++ b/infra/base-images/base-builder/detect_repo_test.py
@@ -23,6 +23,7 @@ import re
import sys
import tempfile
import unittest
+from unittest import mock
import detect_repo
@@ -36,6 +37,33 @@ import test_repos
# pylint: enable=wrong-import-position
+class TestCheckForRepoName(unittest.TestCase):
+ """Tests for check_for_repo_name."""
+
+ @mock.patch('os.path.exists', return_value=True)
+ @mock.patch('detect_repo.execute',
+ return_value=('https://github.com/google/syzkaller/', None))
+ def test_go_get_style_url(self, _, __):
+ """Tests that check_for_repo_name works on repos that were downloaded using
+ go get."""
+ self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller'))
+
+ @mock.patch('os.path.exists', return_value=True)
+ @mock.patch('detect_repo.execute',
+ return_value=('https://github.com/google/syzkaller', None))
+ def test_missing_git_and_slash_url(self, _, __):
+ """Tests that check_for_repo_name works on repos who's URLs do not end in
+ ".git" or "/"."""
+ self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller'))
+
+ @mock.patch('os.path.exists', return_value=True)
+ @mock.patch('detect_repo.execute',
+ return_value=('https://github.com/google/syzkaller.git', None))
+ def test_normal_style_repo_url(self, _, __):
+ """Tests that check_for_repo_name works on normally cloned repos."""
+ self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller'))
+
+
@unittest.skipIf(not os.getenv('INTEGRATION_TESTS'),
'INTEGRATION_TESTS=1 not set')
class DetectRepoIntegrationTest(unittest.TestCase):
diff --git a/infra/base-images/base-builder/precompile_afl b/infra/base-images/base-builder/precompile_afl
new file mode 100755
index 000000000..d6e71f2c3
--- /dev/null
+++ b/infra/base-images/base-builder/precompile_afl
@@ -0,0 +1,35 @@
+#!/bin/bash -eu
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+echo "Precompiling AFLplusplus"
+
+pushd $SRC/aflplusplus > /dev/null
+make clean
+# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it
+# down with sanitizers.
+SAVE_CXXFLAGS=$CXXFLAGS
+SAVE_CFLAGS=$CFLAGS
+unset CXXFLAGS
+unset CFLAGS
+export AFL_IGNORE_UNKNOWN_ENVS=1
+make clean
+AFL_NO_X86=1 PYTHON_INCLUDE=/ make
+make -C utils/aflpp_driver
+
+popd > /dev/null
+
+echo "Done."
diff --git a/infra/base-images/base-builder/precompile_honggfuzz b/infra/base-images/base-builder/precompile_honggfuzz
index 2565bb83f..df6bb2b75 100755
--- a/infra/base-images/base-builder/precompile_honggfuzz
+++ b/infra/base-images/base-builder/precompile_honggfuzz
@@ -15,7 +15,7 @@
#
################################################################################
-echo -n "Precompiling honggfuzz to $PRECOMPILED_DIR..."
+echo "Precompiling honggfuzz"
export BUILD_OSSFUZZ_STATIC=true
PACKAGES=(
@@ -37,10 +37,9 @@ CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD" make
# libhfuzz.a will be added by CC/CXX linker directly during linking,
# but it's defined here to satisfy the build infrastructure
-ar rcs $PRECOMPILED_DIR/honggfuzz.a libhfuzz/*.o libhfcommon/*.o
-cp honggfuzz $PRECOMPILED_DIR/
+ar rcs honggfuzz.a libhfuzz/*.o libhfcommon/*.o
popd > /dev/null
apt-get remove -y --purge ${PACKAGES[@]}
apt-get autoremove -y
-echo " done."
+echo "Done."
diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile
index 928e7934f..3c16a8f3c 100644
--- a/infra/base-images/base-clang/Dockerfile
+++ b/infra/base-images/base-clang/Dockerfile
@@ -25,7 +25,8 @@ RUN apt-get update && apt-get install -y wget sudo && \
chmod +x cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
./cmake-$CMAKE_VERSION-Linux-x86_64.sh --skip-license --prefix="/usr/local" && \
rm cmake-$CMAKE_VERSION-Linux-x86_64.sh && \
- SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo
+ SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo && \
+ rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui
COPY checkout_build_install_llvm.sh /root/
# Keep all steps in the same script to decrease the number of intermediate
diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh
index c5d97d5bf..f6e8ca99c 100755
--- a/infra/base-images/base-clang/checkout_build_install_llvm.sh
+++ b/infra/base-images/base-clang/checkout_build_install_llvm.sh
@@ -15,10 +15,14 @@
#
################################################################################
-NPROC=16 # See issue #4270. The compiler crashes on GCB instance with 32 vCPUs.
+# See issue #4270. The compiler crashes on GCB instance with 32 vCPUs, so when
+# we compile on GCB we want 16 cores. But locally we want more (so use nproc /
+# 2).
+NPROC=$(expr $(nproc) / 2)
-LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev"
-apt-get install -y $LLVM_DEP_PACKAGES
+# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler
+LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev"
+apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
# Checkout
CHECKOUT_RETRIES=10
@@ -60,7 +64,7 @@ function cmake_llvm {
# Use chromium's clang revision
mkdir $SRC/chromium_tools
cd $SRC/chromium_tools
-git clone https://chromium.googlesource.com/chromium/src/tools/clang
+git clone https://chromium.googlesource.com/chromium/src/tools/clang --depth 1
cd clang
LLVM_SRC=$SRC/llvm-project
@@ -89,11 +93,9 @@ fi
git -C $LLVM_SRC checkout $LLVM_REVISION
echo "Using LLVM revision: $LLVM_REVISION"
-# Build & install. We build clang in two stages because gcc can't build a
-# static version of libcxxabi
-# (see https://github.com/google/oss-fuzz/issues/2164).
+# Build & install.
mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1
-cd $WORK/llvm-stage1
+python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1
TARGET_TO_BUILD=
case $(uname -m) in
@@ -111,9 +113,6 @@ esac
PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
-cmake_llvm
-ninja -j $NPROC
-
cd $WORK/llvm-stage2
export CC=$WORK/llvm-stage1/bin/clang
export CXX=$WORK/llvm-stage1/bin/clang++
@@ -175,3 +174,54 @@ rm -rf $LLVM_SRC
rm -rf $SRC/chromium_tools
apt-get remove --purge -y $LLVM_DEP_PACKAGES
apt-get autoremove -y
+
+# Delete unneeded parts of LLVM to reduce image size.
+# See https://github.com/google/oss-fuzz/issues/5170
+LLVM_TOOLS_TMPDIR=/tmp/llvm-tools
+mkdir $LLVM_TOOLS_TMPDIR
+# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR
+mv \
+ /usr/local/bin/llvm-ar \
+ /usr/local/bin/llvm-as \
+ /usr/local/bin/llvm-config \
+ /usr/local/bin/llvm-cov \
+ /usr/local/bin/llvm-objcopy \
+ /usr/local/bin/llvm-profdata \
+ /usr/local/bin/llvm-ranlib \
+ /usr/local/bin/llvm-symbolizer \
+ /usr/local/bin/llvm-undname \
+ $LLVM_TOOLS_TMPDIR
+# Delete remaining llvm- binaries.
+rm -rf /usr/local/bin/llvm-*
+# Restore the llvm- binaries we want to keep.
+mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/
+rm -rf $LLVM_TOOLS_TMPDIR
+
+# Remove binaries from LLVM buld that we don't need.
+rm -f \
+ /usr/local/bin/bugpoint \
+ /usr/local/bin/llc \
+ /usr/local/bin/lli \
+ /usr/local/bin/clang-check \
+ /usr/local/bin/clang-refactor \
+ /usr/local/bin/clang-offload-wrapper \
+ /usr/local/bin/clang-offload-bundler \
+ /usr/local/bin/clang-check \
+ /usr/local/bin/clang-refactor \
+ /usr/local/bin/c-index-test \
+ /usr/local/bin/clang-rename \
+ /usr/local/bin/clang-scan-deps \
+ /usr/local/bin/clang-extdef-mapping \
+ /usr/local/bin/diagtool \
+ /usr/local/bin/sanstats \
+ /usr/local/bin/dsymutil \
+ /usr/local/bin/verify-uselistorder \
+ /usr/local/bin/clang-format
+
+# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the
+# libraries.
+# Note: we need fuzzer_no_main libraries for atheris. Don't delete.
+rm -rf \
+ /usr/local/lib/libclang* \
+ /usr/local/lib/liblld* \
+ /usr/local/lib/cmake/
diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
index f0a264fa7..f847de026 100644..100755
--- a/infra/base-images/base-runner/Dockerfile
+++ b/infra/base-images/base-runner/Dockerfile
@@ -14,30 +14,40 @@
#
################################################################################
+# Build rust stuff in its own image. We only need the resulting binaries.
+# Keeping the rust toolchain in the image wastes 1 GB.
+FROM gcr.io/oss-fuzz-base/base-image as temp-runner-binary-builder
+
+RUN apt-get update && apt-get install -y cargo
+RUN cargo install rustfilt
+
# Using multi-stage build to copy some LLVM binaries needed in the runner image.
FROM gcr.io/oss-fuzz-base/base-clang AS base-clang
+# Real image that will be used later.
FROM gcr.io/oss-fuzz-base/base-image
+COPY --from=temp-runner-binary-builder /root/.cargo/bin/rustfilt /usr/local/bin
+
# Copy the binaries needed for code coverage and crash symbolization.
COPY --from=base-clang /usr/local/bin/llvm-cov \
/usr/local/bin/llvm-profdata \
/usr/local/bin/llvm-symbolizer \
/usr/local/bin/
-# TODO(metzman): Install libc6-i386 lib32gcc1 instead of libc6-dev-i386 for
-# consistency with ClusterFuzz image and to reduce size.
RUN apt-get update && apt-get install -y \
binutils \
file \
fonts-dejavu \
git \
- libc6-dev-i386 \
+ lib32gcc1 \
+ libc6-i386 \
libcap2 \
python3 \
python3-pip \
+ unzip \
wget \
- zip
+ zip --no-install-recommends
RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage /opt/code_coverage && \
pip3 install -r /opt/code_coverage/requirements.txt
@@ -66,7 +76,20 @@ ENV GOPATH /root/go
ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
# Set up Golang coverage modules.
-RUN go get github.com/google/oss-fuzz/infra/go/coverage/...
+COPY gocoverage $GOPATH/gocoverage
+RUN cd $GOPATH/gocoverage && go install ./...
+
+# Install OpenJDK 15 and trim its size by removing unused components.
+ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64
+ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server
+ENV PATH=$PATH:$JAVA_HOME/bin
+
+RUN wget https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz -O /tmp/openjdk-15.0.2_linux-x64_bin.tar.gz && \
+ cd /tmp && \
+ mkdir -p $JAVA_HOME && \
+ tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \
+ rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \
+ rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip
# Do this last to make developing these files easier/faster due to caching.
COPY bad_build_check \
@@ -75,12 +98,11 @@ COPY bad_build_check \
coverage_helper \
dataflow_tracer.py \
download_corpus \
- minijail0 \
+ rcfilt \
reproduce \
run_fuzzer \
- run_minijail \
parse_options.py \
targets_list \
test_all.py \
- test_one \
+ test_one.py \
/usr/local/bin/
diff --git a/infra/base-images/base-runner/bad_build_check b/infra/base-images/base-runner/bad_build_check
index a57a48252..01f8fbbab 100755
--- a/infra/base-images/base-runner/bad_build_check
+++ b/infra/base-images/base-runner/bad_build_check
@@ -301,6 +301,12 @@ function check_mixed_sanitizers {
local result=0
local CALL_INSN=
+ if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then
+ # Sanitizer runtime is linked into the Jazzer driver, so this check does not
+ # apply.
+ return 0
+ fi
+
if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then
# Sanitizer runtime is loaded via LD_PRELOAD, so this check does not apply.
return 0
@@ -312,7 +318,7 @@ function check_mixed_sanitizers {
else
case $(uname -m) in
x86_64)
- CALL_INSN="callq\s+[0-9a-f]+\s+<"
+ CALL_INSN="callq?\s+[0-9a-f]+\s+<"
;;
aarch64)
CALL_INSN="bl\s+[0-9a-f]+\s+<"
@@ -328,6 +334,7 @@ function check_mixed_sanitizers {
local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c)
local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c)
+
if [[ "$SANITIZER" = address ]]; then
check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
result=$?
@@ -340,6 +347,9 @@ function check_mixed_sanitizers {
elif [[ "$SANITIZER" = undefined ]]; then
check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS
result=$?
+ elif [[ "$SANITIZER" = thread ]]; then
+ # TODO(metzman): Implement this.
+ result=0
fi
return $result
@@ -376,6 +386,12 @@ function check_architecture {
local FUZZER=$1
local FUZZER_NAME=$(basename $FUZZER)
+ if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then
+ # The native dependencies of a JVM project are not packaged, but loaded
+ # dynamically at runtime and thus cannot be checked here.
+ return 0;
+ fi
+
if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then
FUZZER=${FUZZER}.pkg
fi
diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage
index 2fcf9e977..a86b00dec 100755
--- a/infra/base-images/base-runner/coverage
+++ b/infra/base-images/base-runner/coverage
@@ -114,13 +114,15 @@ function run_fuzz_target {
function run_go_fuzz_target {
local target=$1
- cd $GOPATH/src
echo "Running go target $target"
export FUZZ_CORPUS_DIR="/corpus/${target}/"
export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf"
- bash $OUT/$target $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log
+ $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log
+ # translate from golangish paths to current absolute paths
+ cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done
+ # cf PATH_EQUIVALENCE_ARGS
+ sed -i 's=/='$OUT'/=' $DUMPS_DIR/$target.profdata
$SYSGOPATH/bin/gocovsum $DUMPS_DIR/$target.profdata > $FUZZER_STATS_DIR/$target.json
- cd $OUT
}
export SYSGOPATH=$GOPATH
@@ -131,7 +133,7 @@ for fuzz_target in $FUZZ_TARGETS; do
if [[ $FUZZING_LANGUAGE == "go" ]]; then
# Continue if not a fuzz target.
if [[ $FUZZING_ENGINE != "none" ]]; then
- grep "go test -run" $fuzz_target > /dev/null 2>&1 || continue
+ grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || continue
fi
run_go_fuzz_target $fuzz_target &
else
@@ -193,7 +195,7 @@ else
# Generate HTML report.
llvm-cov show -format=html -output-dir=$REPORT_ROOT_DIR \
- -Xdemangler c++filt -Xdemangler -n $LLVM_COV_ARGS
+ -Xdemangler rcfilt $LLVM_COV_ARGS
# Export coverage summary in JSON format.
llvm-cov export -summary-only $LLVM_COV_ARGS > $SUMMARY_FILE
diff --git a/infra/base-images/base-runner/gocoverage/go.mod b/infra/base-images/base-runner/gocoverage/go.mod
new file mode 100644
index 000000000..b0b57216e
--- /dev/null
+++ b/infra/base-images/base-runner/gocoverage/go.mod
@@ -0,0 +1,8 @@
+module oss-fuzz.com/gocoverage
+
+go 1.14
+
+require (
+ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5
+ golang.org/x/tools v0.1.0
+)
diff --git a/infra/base-images/base-runner/gocoverage/go.sum b/infra/base-images/base-runner/gocoverage/go.sum
new file mode 100644
index 000000000..3279af3ba
--- /dev/null
+++ b/infra/base-images/base-runner/gocoverage/go.sum
@@ -0,0 +1,30 @@
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo=
+github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
+github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/infra/go/coverage/gocovmerge/LICENSE b/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE
index 455fb1087..455fb1087 100644
--- a/infra/go/coverage/gocovmerge/LICENSE
+++ b/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE
diff --git a/infra/go/coverage/gocovmerge/gocovmerge.go b/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go
index e8099839e..e8099839e 100644
--- a/infra/go/coverage/gocovmerge/gocovmerge.go
+++ b/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go
diff --git a/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go b/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go
new file mode 100644
index 000000000..973b7ae92
--- /dev/null
+++ b/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go
@@ -0,0 +1,147 @@
+package main
+
+import (
+ "encoding/json"
+ "flag"
+ "fmt"
+ "log"
+
+ "go/ast"
+ "go/parser"
+ "go/token"
+
+ "golang.org/x/tools/cover"
+)
+
+type CoverageTotal struct {
+ Count int `json:"count"`
+ Covered int `json:"covered"`
+ Uncovered int `json:"notcovered"`
+ Percent float64 `json:"percent"`
+}
+
+type CoverageTotals struct {
+ Functions CoverageTotal `json:"functions,omitempty"`
+ Lines CoverageTotal `json:"lines,omitempty"`
+ Regions CoverageTotal `json:"regions,omitempty"`
+ Instantiations CoverageTotal `json:"instantiations,omitempty"`
+ Branches CoverageTotal `json:"branches,omitempty"`
+}
+
+type CoverageFile struct {
+ Summary CoverageTotals `json:"summary,omitempty"`
+ Filename string `json:"filename,omitempty"`
+}
+
+type CoverageData struct {
+ Totals CoverageTotals `json:"totals,omitempty"`
+ Files []CoverageFile `json:"files,omitempty"`
+}
+
+type PositionInterval struct {
+ start token.Position
+ end token.Position
+}
+
+type CoverageSummary struct {
+ Data []CoverageData `json:"data,omitempty"`
+ Type string `json:"type,omitempty"`
+ Version string `json:"version,omitempty"`
+}
+
+func isFunctionCovered(s token.Position, e token.Position, blocks []cover.ProfileBlock) bool {
+ for _, b := range blocks {
+ if b.StartLine >= s.Line && b.StartLine <= e.Line && b.EndLine >= s.Line && b.EndLine <= e.Line {
+ if b.Count > 0 {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+func computePercent(s *CoverageTotals) {
+ s.Regions.Percent = float64(100*s.Regions.Covered) / float64(s.Regions.Count)
+ s.Lines.Percent = float64(100*s.Lines.Covered) / float64(s.Lines.Count)
+ s.Functions.Percent = float64(100*s.Functions.Covered) / float64(s.Functions.Count)
+}
+
+func main() {
+ flag.Parse()
+
+ if len(flag.Args()) != 1 {
+ log.Fatalf("needs exactly one argument")
+ }
+ profiles, err := cover.ParseProfiles(flag.Args()[0])
+ if err != nil {
+ log.Fatalf("failed to parse profiles: %v", err)
+ }
+ r := CoverageSummary{}
+ r.Type = "oss-fuzz.go.coverage.json.export"
+ r.Version = "2.0.1"
+ r.Data = make([]CoverageData, 1)
+ for _, p := range profiles {
+ fset := token.NewFileSet() // positions are relative to fset
+ f, err := parser.ParseFile(fset, p.FileName, nil, 0)
+ if err != nil {
+ panic(err)
+ }
+ fileCov := CoverageFile{}
+ fileCov.Filename = p.FileName
+ ast.Inspect(f, func(n ast.Node) bool {
+ switch x := n.(type) {
+ case *ast.FuncLit:
+ startf := fset.Position(x.Pos())
+ endf := fset.Position(x.End())
+ fileCov.Summary.Functions.Count++
+ if isFunctionCovered(startf, endf, p.Blocks) {
+ fileCov.Summary.Functions.Covered++
+ } else {
+ fileCov.Summary.Functions.Uncovered++
+ }
+ case *ast.FuncDecl:
+ startf := fset.Position(x.Pos())
+ endf := fset.Position(x.End())
+ fileCov.Summary.Functions.Count++
+ if isFunctionCovered(startf, endf, p.Blocks) {
+ fileCov.Summary.Functions.Covered++
+ } else {
+ fileCov.Summary.Functions.Uncovered++
+ }
+ }
+ return true
+ })
+
+ for _, b := range p.Blocks {
+ fileCov.Summary.Regions.Count++
+ if b.Count > 0 {
+ fileCov.Summary.Regions.Covered++
+ } else {
+ fileCov.Summary.Regions.Uncovered++
+ }
+
+ fileCov.Summary.Lines.Count += b.NumStmt
+ if b.Count > 0 {
+ fileCov.Summary.Lines.Covered += b.NumStmt
+ } else {
+ fileCov.Summary.Lines.Uncovered += b.NumStmt
+ }
+ }
+ r.Data[0].Totals.Regions.Count += fileCov.Summary.Regions.Count
+ r.Data[0].Totals.Regions.Covered += fileCov.Summary.Regions.Covered
+ r.Data[0].Totals.Regions.Uncovered += fileCov.Summary.Regions.Uncovered
+ r.Data[0].Totals.Lines.Count += fileCov.Summary.Lines.Count
+ r.Data[0].Totals.Lines.Covered += fileCov.Summary.Lines.Covered
+ r.Data[0].Totals.Lines.Uncovered += fileCov.Summary.Lines.Uncovered
+ r.Data[0].Totals.Functions.Count += fileCov.Summary.Functions.Count
+ r.Data[0].Totals.Functions.Covered += fileCov.Summary.Functions.Covered
+ r.Data[0].Totals.Functions.Uncovered += fileCov.Summary.Functions.Uncovered
+
+ computePercent(&fileCov.Summary)
+ r.Data[0].Files = append(r.Data[0].Files, fileCov)
+ }
+
+ computePercent(&r.Data[0].Totals)
+ o, _ := json.Marshal(r)
+ fmt.Printf(string(o))
+}
diff --git a/infra/go/coverage/pprof-merge/LICENSE b/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE
index 8dada3eda..8dada3eda 100644
--- a/infra/go/coverage/pprof-merge/LICENSE
+++ b/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE
diff --git a/infra/go/coverage/pprof-merge/main.go b/infra/base-images/base-runner/gocoverage/pprof-merge/main.go
index f35156403..f35156403 100644
--- a/infra/go/coverage/pprof-merge/main.go
+++ b/infra/base-images/base-runner/gocoverage/pprof-merge/main.go
diff --git a/infra/base-images/base-runner/minijail0 b/infra/base-images/base-runner/minijail0
deleted file mode 100755
index 369e0bbd9..000000000
--- a/infra/base-images/base-runner/minijail0
+++ /dev/null
Binary files differ
diff --git a/infra/base-images/base-runner/rcfilt b/infra/base-images/base-runner/rcfilt
new file mode 100755
index 000000000..1c621100c
--- /dev/null
+++ b/infra/base-images/base-runner/rcfilt
@@ -0,0 +1,21 @@
+#!/bin/bash -u
+# Copyright 2020 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Symbol demangling for both C++ and Rust
+#
+################################################################################
+
+# simply pipe
+rustfilt | c++filt -n
diff --git a/infra/base-images/base-runner/run_fuzzer b/infra/base-images/base-runner/run_fuzzer
index 6464ddc2c..b9bc8d9d6 100755
--- a/infra/base-images/base-runner/run_fuzzer
+++ b/infra/base-images/base-runner/run_fuzzer
@@ -98,22 +98,29 @@ fi
if [[ "$FUZZING_ENGINE" = afl ]]; then
# Set afl++ environment options.
- export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1:symbolize=0"
+ export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1:symbolize=0:detect_odr_violation=0:"
export MSAN_OPTIONS="$MSAN_OPTIONS:exit_code=86:symbolize=0"
export UBSAN_OPTIONS="$UBSAN_OPTIONS:symbolize=0"
export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
export AFL_SKIP_CPUFREQ=1
export AFL_NO_AFFINITY=1
export AFL_FAST_CAL=1
- export AFL_MAP_SIZE=4194304
# If $OUT/afl_cmplog.txt is present this means the target was compiled for
# CMPLOG. So we have to add the proper parameters to afl-fuzz. `-l 2` is
# CMPLOG level 2, which will colorize larger files but not huge files and
# not enable transform analysis unless there have been several cycles without
# any finds.
- test -e $OUT/afl_cmplog.txt && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -l 2 -c $OUT/$FUZZER"
+ test -e "$OUT/afl_cmplog.txt" && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -l 2 -c $OUT/$FUZZER"
+ # If $OUT/afl++.dict we load it as a dictionary for afl-fuzz.
+ test -e "$OUT/afl++.dict" && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -x $OUT/afl++.dict"
+ # Ensure timeout is a bit large than 1sec as some of the OSS-Fuzz fuzzers
+ # are slower than this.
+ AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -t 5000+"
# AFL expects at least 1 file in the input dir.
echo input > ${CORPUS_DIR}/input
+ echo afl++ setup:
+ env|grep AFL_
+ cat "$OUT/afl_options.txt"
CMD_LINE="$OUT/afl-fuzz $AFL_FUZZER_ARGS -i $CORPUS_DIR -o $FUZZER_OUT $(get_dictionary) $* -- $OUT/$FUZZER"
elif [[ "$FUZZING_ENGINE" = honggfuzz ]]; then
diff --git a/infra/base-images/base-runner/run_minijail b/infra/base-images/base-runner/run_minijail
deleted file mode 100755
index bf950b1e7..000000000
--- a/infra/base-images/base-runner/run_minijail
+++ /dev/null
@@ -1,59 +0,0 @@
-#!/bin/bash -eu
-# Copyright 2017 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-################################################################################
-
-rm -rf /tmp/chroot
-mkdir /tmp/chroot
-
-mkdir /tmp/chroot/lib
-mkdir /tmp/chroot/lib64
-mkdir /tmp/chroot/lib32
-
-mkdir /tmp/chroot/usr
-mkdir /tmp/chroot/usr/lib
-mkdir /tmp/chroot/usr/lib32
-
-mkdir /tmp/chroot/dev
-mknod -m 666 /tmp/chroot/dev/null c 1 3
-mknod -m 666 /tmp/chroot/dev/urandom c 1 9
-
-mkdir /tmp/chroot/proc
-mkdir /tmp/chroot/tmp
-
-mkdir /tmp/chroot/bin
-cp /bin/sh /tmp/chroot/bin/sh
-cp $(which llvm-symbolizer) /tmp/chroot/bin/llvm-symbolizer
-
-FULL_EXE_PATH=$(readlink -f $1)
-EXE_DIR=$(dirname $FULL_EXE_PATH)
-mkdir -p /tmp/chroot/$EXE_DIR
-
-shift
-
-echo 'Running:'
-echo minijail0 -U -m \"0 $UID 1\" -T static \
- -c 0 -n -v -p -l -I \
- -k proc,/proc,proc,1 -P /tmp/chroot \
- -b /lib,/lib,0 -b /lib64,/lib64,0 -b /lib32,/lib32,0 -b /usr/lib,/usr/lib,0 \
- -b /usr/lib32,/usr/lib32,0 -b /tmp,/tmp,1 \
- -b $EXE_DIR,$EXE_DIR,0 $FULL_EXE_PATH $@
-
-minijail0 -U -m "0 $UID 1" -T static \
- -c 0 -n -v -p -l -I \
- -k proc,/proc,proc,1 -P /tmp/chroot \
- -b /lib,/lib,0 -b /lib64,/lib64,0 -b /lib32,/lib32,0 -b /usr/lib,/usr/lib,0 \
- -b /usr/lib32,/usr/lib32,0 -b /tmp,/tmp,1 \
- -b $EXE_DIR,$EXE_DIR,0 $FULL_EXE_PATH $@
diff --git a/infra/base-images/base-runner/test_all.py b/infra/base-images/base-runner/test_all.py
index 360da0345..925ebde69 100755
--- a/infra/base-images/base-runner/test_all.py
+++ b/infra/base-images/base-runner/test_all.py
@@ -78,11 +78,16 @@ def find_fuzz_targets(directory, fuzzing_language):
continue
if filename.startswith('afl-'):
continue
+ if filename.startswith('jazzer_'):
+ continue
if not os.path.isfile(path):
continue
if not os.stat(path).st_mode & EXECUTABLE:
continue
- if fuzzing_language != 'python' and not is_elf(path):
+ # Fuzz targets are expected to be ELF binaries for languages other than
+ # Python and Java.
+ if (fuzzing_language != 'python' and fuzzing_language != 'jvm' and
+ not is_elf(path)):
continue
if os.getenv('FUZZING_ENGINE') != 'none':
with open(path, 'rb') as file_handle:
diff --git a/infra/base-images/base-runner/test_one b/infra/base-images/base-runner/test_one
deleted file mode 100755
index 23b7fd932..000000000
--- a/infra/base-images/base-runner/test_one
+++ /dev/null
@@ -1,58 +0,0 @@
-#!/bin/bash -u
-# Copyright 2020 Google Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-################################################################################
-
-# Wrapper around bad_build_check that moves the /out directory to /tmp/not-out.
-# This is useful when bad_build_check isn't called from test_all which does the
-# same thing.
-
-function main {
- # Move the directory the fuzzer is located in to somewhere that doesn't exist
- # on the builder to make it more likely that hardcoding /out fails here (since
- # it will fail on ClusterFuzz).
- local fuzzer=$1
- fuzzer=$(realpath $fuzzer)
- local initial_fuzzer_dir=$(dirname $fuzzer)
-
- local tmp_fuzzer_dir=/tmp/not-out
- rm -rf $tmp_fuzzer_dir
- mkdir $tmp_fuzzer_dir
- # Move the contents of $initial_fuzzer_dir rather than the directory itself in
- # case it is a mount.
- mv $initial_fuzzer_dir/* $tmp_fuzzer_dir
- fuzzer="$tmp_fuzzer_dir/$(basename $fuzzer)"
-
- # Change OUT to the temporary fuzzer dir.
- local initial_out=$OUT
- export OUT=$tmp_fuzzer_dir
-
- bad_build_check $fuzzer
- returncode=$?
-
- # Restore OUT and $initial_fuzzer_dir
- export OUT=$initial_out
- mv $tmp_fuzzer_dir/* $initial_fuzzer_dir
-
- return $returncode
-}
-
-if [ $# -ne 1 ]; then
- echo "Usage: $0 <fuzz_target_binary>"
- exit 1
-fi
-
-main $1
-exit $?
diff --git a/infra/base-images/base-runner/test_one.py b/infra/base-images/base-runner/test_one.py
new file mode 100755
index 000000000..9bdb75faf
--- /dev/null
+++ b/infra/base-images/base-runner/test_one.py
@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+"""Does bad_build_check on a fuzz target in $OUT."""
+import os
+import sys
+
+import test_all
+
+
+def test_one(fuzz_target):
+ """Does bad_build_check on one fuzz target. Returns True on success."""
+ with test_all.use_different_out_dir():
+ fuzz_target_path = os.path.join(os.environ['OUT'], fuzz_target)
+ return test_all.do_bad_build_check(fuzz_target_path).returncode == 0
+
+
+def main():
+ """Does bad_build_check on one fuzz target. Returns 1 on failure, 0 on
+ success."""
+ if len(sys.argv) != 2:
+ print('Usage: %d <fuzz_target>', sys.argv[0])
+ return 1
+
+ fuzz_target_binary = sys.argv[1]
+ return 0 if test_one(fuzz_target_binary) else 1
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/infra/base-images/base-sanitizer-libs-builder/msan_build.py b/infra/base-images/base-sanitizer-libs-builder/msan_build.py
index 928b1a596..5ea00ab10 100755
--- a/infra/base-images/base-sanitizer-libs-builder/msan_build.py
+++ b/infra/base-images/base-sanitizer-libs-builder/msan_build.py
@@ -73,7 +73,9 @@ def SetUpEnvironment(work_dir):
dpkg_host_architecture = wrapper_utils.DpkgHostArchitecture()
wrapper_utils.CreateSymlinks(
- compiler_wrapper_path, bin_dir, [
+ compiler_wrapper_path,
+ bin_dir,
+ [
'clang',
'clang++',
# Not all build rules respect $CC/$CXX, so make additional symlinks.
@@ -101,41 +103,35 @@ def SetUpEnvironment(work_dir):
env['DPKG_GENSYMBOLS_CHECK_LEVEL'] = '0'
# debian/rules can set DPKG_GENSYMBOLS_CHECK_LEVEL explicitly, so override it.
- gen_symbols_wrapper = (
- '#!/bin/sh\n'
- 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n'
- '/usr/bin/dpkg-gensymbols "$@"\n')
+ gen_symbols_wrapper = ('#!/bin/sh\n'
+ 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n'
+ '/usr/bin/dpkg-gensymbols "$@"\n')
- wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols',
- gen_symbols_wrapper)
+ wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols', gen_symbols_wrapper)
# Install no-op strip binaries.
- no_op_strip = ('#!/bin/sh\n'
- 'exit 0\n')
- wrapper_utils.InstallWrapper(
- bin_dir, 'strip', no_op_strip,
- [dpkg_host_architecture + '-strip'])
+ no_op_strip = ('#!/bin/sh\n' 'exit 0\n')
+ wrapper_utils.InstallWrapper(bin_dir, 'strip', no_op_strip,
+ [dpkg_host_architecture + '-strip'])
env['PATH'] = bin_dir + ':' + os.environ['PATH']
# nocheck doesn't disable override_dh_auto_test. So we have this hack to try
# to disable "make check" or "make test" invocations.
- make_wrapper = (
- '#!/bin/bash\n'
- 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n'
- ' exit 0\n'
- 'fi\n'
- '/usr/bin/make "$@"\n')
- wrapper_utils.InstallWrapper(bin_dir, 'make',
- make_wrapper)
+ make_wrapper = ('#!/bin/bash\n'
+ 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n'
+ ' exit 0\n'
+ 'fi\n'
+ '/usr/bin/make "$@"\n')
+ wrapper_utils.InstallWrapper(bin_dir, 'make', make_wrapper)
# Prevent entire build from failing because of bugs/uninstrumented in tools
# that are part of the build.
msan_log_dir = os.path.join(work_dir, 'msan')
os.mkdir(msan_log_dir)
msan_log_path = os.path.join(msan_log_dir, 'log')
- env['MSAN_OPTIONS'] = (
- 'halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + msan_log_path)
+ env['MSAN_OPTIONS'] = ('halt_on_error=0:exitcode=0:report_umrs=0:log_path=' +
+ msan_log_path)
# Increase maximum stack size to prevent tests from failing.
limit = 128 * 1024 * 1024
@@ -207,7 +203,7 @@ def ExtractLibraries(deb_paths, work_directory, output_directory):
target_file_path = os.path.join(output_directory, rel_file_path)
extracted.append(target_file_path)
-
+
if os.path.lexists(target_file_path):
os.remove(target_file_path)
@@ -215,8 +211,8 @@ def ExtractLibraries(deb_paths, work_directory, output_directory):
link_path = os.readlink(file_path)
if os.path.isabs(link_path):
# Make absolute links relative.
- link_path = os.path.relpath(
- link_path, os.path.join('/', rel_directory))
+ link_path = os.path.relpath(link_path,
+ os.path.join('/', rel_directory))
os.symlink(link_path, target_file_path)
else:
@@ -244,8 +240,8 @@ def GetPackage(package_name):
def PatchRpath(path, output_directory):
"""Patch rpath to be relative to $ORIGIN."""
try:
- rpaths = subprocess.check_output(
- ['patchelf', '--print-rpath', path]).strip()
+ rpaths = subprocess.check_output(['patchelf', '--print-rpath',
+ path]).strip()
except subprocess.CalledProcessError:
return
@@ -262,15 +258,13 @@ def PatchRpath(path, output_directory):
processed_rpath.append(rpath)
continue
- processed_rpath.append(os.path.join(
- '$ORIGIN',
- os.path.relpath(rpath, rel_directory)))
+ processed_rpath.append(
+ os.path.join('$ORIGIN', os.path.relpath(rpath, rel_directory)))
processed_rpath = ':'.join(processed_rpath)
print('Patching rpath for', path, 'to', processed_rpath)
subprocess.check_call(
- ['patchelf', '--force-rpath', '--set-rpath',
- processed_rpath, path])
+ ['patchelf', '--force-rpath', '--set-rpath', processed_rpath, path])
def _CollectDependencies(apt_cache, pkg, cache, dependencies):
@@ -331,7 +325,11 @@ def GetBuildList(package_name):
class MSanBuilder(object):
"""MSan builder."""
- def __init__(self, debug=False, log_path=None, work_dir=None, no_track_origins=False):
+ def __init__(self,
+ debug=False,
+ log_path=None,
+ work_dir=None,
+ no_track_origins=False):
self.debug = debug
self.log_path = log_path
self.work_dir = work_dir
@@ -396,19 +394,24 @@ class MSanBuilder(object):
extracted_paths = ExtractLibraries(deb_paths, self.work_dir,
extract_directory)
for extracted_path in extracted_paths:
- if not os.path.islink(extracted_path):
- PatchRpath(extracted_path, extract_directory)
+ if os.path.islink(extracted_path):
+ continue
+ if os.path.basename(extracted_path) == 'llvm-symbolizer':
+ continue
+ PatchRpath(extracted_path, extract_directory)
def main():
parser = argparse.ArgumentParser('msan_build.py', description='MSan builder.')
parser.add_argument('package_names', nargs='+', help='Name of the packages.')
parser.add_argument('output_dir', help='Output directory.')
- parser.add_argument('--create-subdirs', action='store_true',
+ parser.add_argument('--create-subdirs',
+ action='store_true',
help=('Create subdirectories in the output '
'directory for each package.'))
parser.add_argument('--work-dir', help='Work directory.')
- parser.add_argument('--no-build-deps', action='store_true',
+ parser.add_argument('--no-build-deps',
+ action='store_true',
help='Don\'t build dependencies.')
parser.add_argument('--debug', action='store_true', help='Enable debug mode.')
parser.add_argument('--log-path', help='Log path for debugging.')
@@ -445,7 +448,8 @@ def main():
for package_name in package_names:
print('\t', package_name)
- with MSanBuilder(debug=args.debug, log_path=args.log_path,
+ with MSanBuilder(debug=args.debug,
+ log_path=args.log_path,
work_dir=args.work_dir,
no_track_origins=args.no_track_origins) as builder:
for package_name in package_names:
diff --git a/infra/bisector.py b/infra/bisector.py
index dc4a470d5..1438d0de9 100644
--- a/infra/bisector.py
+++ b/infra/bisector.py
@@ -189,6 +189,8 @@ def _bisect(bisect_type, old_commit, new_commit, test_case_path, fuzz_target,
bisect_repo_manager = repo_manager.RepoManager(
os.path.join(host_src_dir, os.path.basename(repo_path)))
+ bisect_repo_manager.fetch_all_remotes()
+
commit_list = bisect_repo_manager.get_commit_list(new_commit, old_commit)
old_idx = len(commit_list) - 1
diff --git a/infra/build/functions/build_and_run_coverage.py b/infra/build/functions/build_and_run_coverage.py
index 71d7338f9..cc2de5a32 100644
--- a/infra/build/functions/build_and_run_coverage.py
+++ b/infra/build/functions/build_and_run_coverage.py
@@ -48,7 +48,7 @@ LATEST_REPORT_INFO_CONTENT_TYPE = 'application/json'
UPLOAD_URL_FORMAT = 'gs://' + COVERAGE_BUCKET_NAME + '/{project}/{type}/{date}'
# Languages from project.yaml that have code coverage support.
-LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go']
+LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust']
def usage():
diff --git a/infra/build/functions/requirements.txt b/infra/build/functions/requirements.txt
index 8d07d82ff..a60a03283 100644
--- a/infra/build/functions/requirements.txt
+++ b/infra/build/functions/requirements.txt
@@ -16,7 +16,7 @@
Brotli==1.0.9
hiredis==1.1.0
-PyYaml==5.2
+PyYaml==5.4
PyGithub==1.51
grpcio==1.29.0
google-auth==1.21.1
diff --git a/infra/build/functions/update_build_status.py b/infra/build/functions/update_build_status.py
index b5f955227..af65a41ab 100644
--- a/infra/build/functions/update_build_status.py
+++ b/infra/build/functions/update_build_status.py
@@ -195,7 +195,10 @@ def update_build_badges(project, last_build_successful,
last_coverage_build_successful):
"""Upload badges of given project."""
badge = 'building'
- if not last_coverage_build_successful:
+ # last_coverage_build_successful is False if there was an unsuccessful build
+ # and None if the target does not support coverage (e.g. Python or Java
+ # targets).
+ if last_coverage_build_successful is False:
badge = 'coverage_failing'
if not last_build_successful:
badge = 'failing'
@@ -289,12 +292,16 @@ def update_badges():
futures = []
with ndb.Client().context():
for project in Project.query():
- if (project.name not in project_build_statuses or
- project.name not in coverage_build_statuses):
+ if project.name not in project_build_statuses:
continue
+ # Certain projects (e.g. JVM and Python) do not have any coverage
+ # builds, but should still receive a badge.
+ coverage_build_status = None
+ if project.name in coverage_build_statuses:
+ coverage_build_status = coverage_build_statuses[project.name]
futures.append(
executor.submit(update_build_badges, project.name,
project_build_statuses[project.name],
- coverage_build_statuses[project.name]))
+ coverage_build_status))
concurrent.futures.wait(futures)
diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py
index 9f29e420e..b2130ea85 100644
--- a/infra/build_specified_commit.py
+++ b/infra/build_specified_commit.py
@@ -200,6 +200,7 @@ def build_fuzzers_from_commit(commit,
# Re-copy /src for a clean checkout every time.
copy_src_from_docker(build_data.project_name,
os.path.dirname(host_src_path))
+ build_repo_manager.fetch_all_remotes()
projects_dir = os.path.join('projects', build_data.project_name)
dockerfile_path = os.path.join(projects_dir, 'Dockerfile')
diff --git a/infra/build_specified_commit_test.py b/infra/build_specified_commit_test.py
index 916b31885..a86504580 100644
--- a/infra/build_specified_commit_test.py
+++ b/infra/build_specified_commit_test.py
@@ -95,6 +95,10 @@ class BuildImageIntegrationTest(unittest.TestCase):
def test_detect_main_repo_from_name(self):
"""Test the detect main repo function from build specific commit module."""
for example_repo in test_repos.TEST_REPOS:
+ if example_repo.project_name == 'gonids':
+ # It's unclear how this test ever passed, but we can't infer the repo
+ # because gonids doesn't really check it out, it uses "go get".
+ continue
repo_origin, repo_name = build_specified_commit.detect_main_repo(
example_repo.project_name, repo_name=example_repo.git_repo_name)
self.assertEqual(repo_origin, example_repo.git_url)
diff --git a/infra/ci/build.py b/infra/ci/build.py
index f71799bb2..addeb7879 100755
--- a/infra/ci/build.py
+++ b/infra/ci/build.py
@@ -32,7 +32,7 @@ DEFAULT_ENGINES = ['afl', 'honggfuzz', 'libfuzzer']
DEFAULT_SANITIZERS = ['address', 'undefined']
# Languages from project.yaml that have code coverage support.
-LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go']
+LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust']
def get_changed_files_output():
diff --git a/infra/ci/requirements.txt b/infra/ci/requirements.txt
index 48d2ae758..f0a8be0b5 100644
--- a/infra/ci/requirements.txt
+++ b/infra/ci/requirements.txt
@@ -4,5 +4,5 @@ pyfakefs==4.1.0
pylint==2.5.3
pytest==6.2.1
pytest-xdist==2.2.0
-PyYAML==5.3.1
+PyYAML==5.4
yapf==0.30.0
diff --git a/infra/cifuzz/actions/build_fuzzers/action.yml b/infra/cifuzz/actions/build_fuzzers/action.yml
index 2919db40e..835b7b430 100644
--- a/infra/cifuzz/actions/build_fuzzers/action.yml
+++ b/infra/cifuzz/actions/build_fuzzers/action.yml
@@ -5,6 +5,10 @@ inputs:
oss-fuzz-project-name:
description: 'Name of the corresponding OSS-Fuzz project.'
required: true
+ language:
+ description: 'Programming language project is written in.'
+ required: false
+ default: 'c++'
dry-run:
description: 'If set, run the action without actually reporting a failure.'
default: false
@@ -20,13 +24,20 @@ inputs:
build-integration-path:
description: "The path to the the project's build integration."
required: false
+ bad-build-check:
+ description: "Whether or not OSS-Fuzz's check for bad builds should be done."
+ required: false
+ default: true
runs:
using: 'docker'
image: '../../../build_fuzzers.Dockerfile'
env:
OSS_FUZZ_PROJECT_NAME: ${{ inputs.oss-fuzz-project-name }}
+ LANGUAGE: ${{ inputs.language }}
DRY_RUN: ${{ inputs.dry-run}}
ALLOWED_BROKEN_TARGETS_PERCENTAGE: ${{ inputs.allowed-broken-targets-percentage}}
SANITIZER: ${{ inputs.sanitizer }}
PROJECT_SRC_PATH: ${{ inputs.project-src-path }}
BUILD_INTEGRATION_PATH: ${{ inputs.build-integration-path }}
+ LOW_DISK_SPACE: 'True'
+ BAD_BUILD_CHECK: ${{ inputs.bad-build-check }}
diff --git a/infra/cifuzz/actions/run_fuzzers/action.yml b/infra/cifuzz/actions/run_fuzzers/action.yml
index 582133c74..d1c03c833 100644
--- a/infra/cifuzz/actions/run_fuzzers/action.yml
+++ b/infra/cifuzz/actions/run_fuzzers/action.yml
@@ -5,6 +5,10 @@ inputs:
oss-fuzz-project-name:
description: 'The OSS-Fuzz project name.'
required: true
+ language:
+ description: 'Programming language project is written in.'
+ required: false
+ default: 'c++'
fuzz-seconds:
description: 'The total time allotted for fuzzing in seconds.'
required: true
@@ -31,6 +35,7 @@ runs:
image: '../../../run_fuzzers.Dockerfile'
env:
OSS_FUZZ_PROJECT_NAME: ${{ inputs.oss-fuzz-project-name }}
+ LANGUAGE: ${{ inputs.language }}
FUZZ_SECONDS: ${{ inputs.fuzz-seconds }}
DRY_RUN: ${{ inputs.dry-run}}
SANITIZER: ${{ inputs.sanitizer }}
@@ -39,3 +44,4 @@ runs:
# for running because we use it to distinguish OSS-Fuzz from non-OSS-Fuzz.
# We should do something explicit instead.
BUILD_INTEGRATION_PATH: ${{ inputs.build-integration-path }}
+ LOW_DISK_SPACE: 'True'
diff --git a/infra/cifuzz/affected_fuzz_targets_test.py b/infra/cifuzz/affected_fuzz_targets_test.py
index 72e6d266c..05f27c072 100644
--- a/infra/cifuzz/affected_fuzz_targets_test.py
+++ b/infra/cifuzz/affected_fuzz_targets_test.py
@@ -30,15 +30,15 @@ EXAMPLE_PROJECT = 'example'
EXAMPLE_FILE_CHANGED = 'test.txt'
-TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'test_files')
+TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ 'test_data')
class RemoveUnaffectedFuzzTargets(unittest.TestCase):
"""Tests remove_unaffected_fuzzers."""
- TEST_FUZZER_1 = os.path.join(TEST_FILES_PATH, 'out', 'example_crash_fuzzer')
- TEST_FUZZER_2 = os.path.join(TEST_FILES_PATH, 'out', 'example_nocrash_fuzzer')
+ TEST_FUZZER_1 = os.path.join(TEST_DATA_PATH, 'out', 'example_crash_fuzzer')
+ TEST_FUZZER_2 = os.path.join(TEST_DATA_PATH, 'out', 'example_nocrash_fuzzer')
# yapf: disable
@parameterized.parameterized.expand([
diff --git a/infra/cifuzz/build_fuzzers.py b/infra/cifuzz/build_fuzzers.py
index a4342a413..78180b52b 100644
--- a/infra/cifuzz/build_fuzzers.py
+++ b/infra/cifuzz/build_fuzzers.py
@@ -20,6 +20,7 @@ import sys
import affected_fuzz_targets
import continuous_integration
+import docker
# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -77,7 +78,8 @@ class Builder: # pylint: disable=too-many-instance-attributes
def build_fuzzers(self):
"""Moves the source code we want to fuzz into the project builder and builds
the fuzzers from that source code. Returns True on success."""
- docker_args = get_common_docker_args(self.config.sanitizer)
+ docker_args = get_common_docker_args(self.config.sanitizer,
+ self.config.language)
container = utils.get_container_name()
if container:
@@ -93,7 +95,7 @@ class Builder: # pylint: disable=too-many-instance-attributes
self.handle_msan_prebuild(container)
docker_args.extend([
- 'gcr.io/oss-fuzz/' + self.config.project_name,
+ docker.get_project_image_name(self.config.project_name),
'/bin/bash',
'-c',
])
@@ -118,8 +120,7 @@ class Builder: # pylint: disable=too-many-instance-attributes
helper.docker_run([
'--volumes-from', container, '-e',
'WORK={work_dir}'.format(work_dir=self.work_dir),
- 'gcr.io/oss-fuzz-base/base-sanitizer-libs-builder', 'patch_build.py',
- '/out'
+ docker.MSAN_LIBS_BUILDER_TAG, 'patch_build.py', '/out'
])
def handle_msan_prebuild(self, container):
@@ -127,8 +128,8 @@ class Builder: # pylint: disable=too-many-instance-attributes
returns docker arguments to use that directory for MSAN libs."""
logging.info('Copying MSAN libs.')
helper.docker_run([
- '--volumes-from', container, 'gcr.io/oss-fuzz-base/msan-libs-builder',
- 'bash', '-c', 'cp -r /msan {work_dir}'.format(work_dir=self.work_dir)
+ '--volumes-from', container, docker.MSAN_LIBS_BUILDER_TAG, 'bash', '-c',
+ 'cp -r /msan {work_dir}'.format(work_dir=self.work_dir)
])
def build(self):
@@ -185,7 +186,7 @@ def build_fuzzers(config):
return builder.build()
-def get_common_docker_args(sanitizer):
+def get_common_docker_args(sanitizer, language):
"""Returns a list of common docker arguments."""
return [
'--cap-add',
@@ -199,12 +200,13 @@ def get_common_docker_args(sanitizer):
'-e',
'CIFUZZ=True',
'-e',
- 'FUZZING_LANGUAGE=c++', # FIXME: Add proper support.
+ 'FUZZING_LANGUAGE=' + language,
]
def check_fuzzer_build(out_dir,
- sanitizer='address',
+ sanitizer,
+ language,
allowed_broken_targets_percentage=None):
"""Checks the integrity of the built fuzzers.
@@ -222,7 +224,7 @@ def check_fuzzer_build(out_dir,
logging.error('No fuzzers found in out directory: %s.', out_dir)
return False
- command = get_common_docker_args(sanitizer)
+ command = get_common_docker_args(sanitizer, language)
if allowed_broken_targets_percentage is not None:
command += [
@@ -236,7 +238,7 @@ def check_fuzzer_build(out_dir,
command += ['-e', 'OUT=' + out_dir, '--volumes-from', container]
else:
command += ['-v', '%s:/out' % out_dir]
- command.extend(['-t', 'gcr.io/oss-fuzz-base/base-runner', 'test_all.py'])
+ command.extend(['-t', docker.BASE_RUNNER_TAG, 'test_all.py'])
exit_code = helper.docker_run(command)
logging.info('check fuzzer build exit code: %d', exit_code)
if exit_code:
diff --git a/infra/cifuzz/build_fuzzers_entrypoint.py b/infra/cifuzz/build_fuzzers_entrypoint.py
index 9c4b98215..04f562068 100644
--- a/infra/cifuzz/build_fuzzers_entrypoint.py
+++ b/infra/cifuzz/build_fuzzers_entrypoint.py
@@ -72,10 +72,16 @@ def main():
return returncode
out_dir = os.path.join(config.workspace, 'out')
+
+ if not config.bad_build_check:
+ # If we've gotten to this point and we don't need to do bad_build_check,
+ # then the build has succeeded.
+ returncode = 0
# yapf: disable
- if build_fuzzers.check_fuzzer_build(
+ elif build_fuzzers.check_fuzzer_build(
out_dir,
- sanitizer=config.sanitizer,
+ config.sanitizer,
+ config.language,
allowed_broken_targets_percentage=config.allowed_broken_targets_percentage
):
# yapf: enable
diff --git a/infra/cifuzz/build_fuzzers_test.py b/infra/cifuzz/build_fuzzers_test.py
index 2d27356d2..298778867 100644
--- a/infra/cifuzz/build_fuzzers_test.py
+++ b/infra/cifuzz/build_fuzzers_test.py
@@ -36,9 +36,9 @@ import test_helpers
# https://github.com/google/oss-fuzz/tree/master/projects/example project.
EXAMPLE_PROJECT = 'example'
-# Location of files used for testing.
-TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'test_files')
+# Location of data used for testing.
+TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ 'test_data')
# An example fuzzer that triggers an crash.
# Binary is a copy of the example project's do_stuff_fuzzer and can be
@@ -251,10 +251,13 @@ class BuildFuzzersIntegrationTest(unittest.TestCase):
class CheckFuzzerBuildTest(unittest.TestCase):
"""Tests the check_fuzzer_build function in the cifuzz module."""
+ SANITIZER = 'address'
+ LANGUAGE = 'c++'
+
def setUp(self):
self.tmp_dir_obj = tempfile.TemporaryDirectory()
self.test_files_path = os.path.join(self.tmp_dir_obj.name, 'test_files')
- shutil.copytree(TEST_FILES_PATH, self.test_files_path)
+ shutil.copytree(TEST_DATA_PATH, self.test_files_path)
def tearDown(self):
self.tmp_dir_obj.cleanup()
@@ -262,23 +265,31 @@ class CheckFuzzerBuildTest(unittest.TestCase):
def test_correct_fuzzer_build(self):
"""Checks check_fuzzer_build function returns True for valid fuzzers."""
test_fuzzer_dir = os.path.join(self.test_files_path, 'out')
- self.assertTrue(build_fuzzers.check_fuzzer_build(test_fuzzer_dir))
+ self.assertTrue(
+ build_fuzzers.check_fuzzer_build(test_fuzzer_dir, self.SANITIZER,
+ self.LANGUAGE))
def test_not_a_valid_fuzz_path(self):
"""Tests that False is returned when a bad path is given."""
- self.assertFalse(build_fuzzers.check_fuzzer_build('not/a/valid/path'))
+ self.assertFalse(
+ build_fuzzers.check_fuzzer_build('not/a/valid/path', self.SANITIZER,
+ self.LANGUAGE))
def test_not_a_valid_fuzzer(self):
"""Checks a directory that exists but does not have fuzzers is False."""
- self.assertFalse(build_fuzzers.check_fuzzer_build(self.test_files_path))
+ self.assertFalse(
+ build_fuzzers.check_fuzzer_build(self.test_files_path, self.SANITIZER,
+ self.LANGUAGE))
@mock.patch('helper.docker_run')
def test_allow_broken_fuzz_targets_percentage(self, mocked_docker_run):
"""Tests that ALLOWED_BROKEN_TARGETS_PERCENTAGE is set when running
docker if passed to check_fuzzer_build."""
mocked_docker_run.return_value = 0
- test_fuzzer_dir = os.path.join(TEST_FILES_PATH, 'out')
+ test_fuzzer_dir = os.path.join(TEST_DATA_PATH, 'out')
build_fuzzers.check_fuzzer_build(test_fuzzer_dir,
+ self.SANITIZER,
+ self.LANGUAGE,
allowed_broken_targets_percentage='0')
self.assertIn('-e ALLOWED_BROKEN_TARGETS_PERCENTAGE=0',
' '.join(mocked_docker_run.call_args[0][0]))
diff --git a/infra/cifuzz/cifuzz-base/Dockerfile b/infra/cifuzz/cifuzz-base/Dockerfile
index 0aee3b2cf..e0599dbbe 100644
--- a/infra/cifuzz/cifuzz-base/Dockerfile
+++ b/infra/cifuzz/cifuzz-base/Dockerfile
@@ -14,25 +14,19 @@
#
################################################################################
+# Don't bother with a slimmer base image.
+# When we pull base-builder to build project builder image we need to pull
+# ubuntu:16.04 anyway. So in the long run we probably would waste time if
+# we pulled something like alpine here instead.
FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y git \
- apt-transport-https \
- ca-certificates \
- curl \
- gnupg2 \
- software-properties-common \
- python3
+RUN apt-get update && \
+ apt-get install ca-certificates wget python3 git-core --no-install-recommends -y && \
+ wget https://download.docker.com/linux/ubuntu/dists/xenial/pool/stable/amd64/docker-ce-cli_20.10.5~3-0~ubuntu-xenial_amd64.deb -O /tmp/docker-ce.deb && \
+ dpkg -i /tmp/docker-ce.deb && rm /tmp/docker-ce.deb && \
+ apt-get remove wget -y --purge
-RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - && apt-key fingerprint 0EBFCD88
-RUN add-apt-repository \
- "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
- xenial \
- stable"
-
-RUN apt-get update && apt-get install docker-ce docker-ce-cli containerd.io -y
-
ENV OSS_FUZZ_ROOT=/opt/oss-fuzz
ADD . ${OSS_FUZZ_ROOT}
RUN rm -rf ${OSS_FUZZ_ROOT}/infra \ No newline at end of file
diff --git a/infra/cifuzz/config_utils.py b/infra/cifuzz/config_utils.py
index fd1871497..ad2cd36c6 100644
--- a/infra/cifuzz/config_utils.py
+++ b/infra/cifuzz/config_utils.py
@@ -18,14 +18,16 @@ import enum
import os
import json
+import environment
+
def _get_project_repo_name():
- return os.path.basename(os.getenv('GITHUB_REPOSITORY', ''))
+ return os.path.basename(environment.get('GITHUB_REPOSITORY', ''))
def _get_pr_ref(event):
if event == 'pull_request':
- return os.getenv('GITHUB_REF')
+ return environment.get('GITHUB_REF')
return None
@@ -40,7 +42,7 @@ def _get_project_name():
def _is_dry_run():
"""Returns True if configured to do a dry run."""
- return os.getenv('DRY_RUN', 'false').lower() == 'true'
+ return environment.get_bool('DRY_RUN', 'false')
def get_project_src_path(workspace):
@@ -62,6 +64,19 @@ def get_project_src_path(workspace):
return os.path.join(workspace, path)
+DEFAULT_LANGUAGE = 'c++'
+
+
+def _get_language():
+ """Returns the project language."""
+ # Get language from environment. We took this approach because the convenience
+ # given to OSS-Fuzz users by not making them specify the language again (and
+ # getting it from the project.yaml) is outweighed by the complexity in
+ # implementing this. A lot of the complexity comes from our unittests not
+ # setting a proper projet at this point.
+ return os.getenv('LANGUAGE', DEFAULT_LANGUAGE)
+
+
# pylint: disable=too-few-public-methods,too-many-instance-attributes
@@ -81,14 +96,22 @@ class BaseConfig:
self.dry_run = _is_dry_run()
self.sanitizer = _get_sanitizer()
self.build_integration_path = os.getenv('BUILD_INTEGRATION_PATH')
+ self.language = _get_language()
event_path = os.getenv('GITHUB_EVENT_PATH')
self.is_github = bool(event_path)
logging.debug('Is github: %s.', self.is_github)
+ # TODO(metzman): Parse env like we do in ClusterFuzz.
+ self.low_disk_space = environment.get('LOW_DISK_SPACE', False)
+
+ @property
+ def is_internal(self):
+ """Returns True if this is an OSS-Fuzz project."""
+ return not self.build_integration_path
@property
def platform(self):
"""Returns the platform CIFuzz is runnning on."""
- if self.build_integration_path:
+ if not self.is_internal:
return self.Platform.EXTERNAL_GITHUB
if self.is_github:
return self.Platform.INTERNAL_GITHUB
@@ -149,6 +172,7 @@ class BuildFuzzersConfig(BaseConfig):
self.allowed_broken_targets_percentage = os.getenv(
'ALLOWED_BROKEN_TARGETS_PERCENTAGE')
+ self.bad_build_check = environment.get_bool('BAD_BUILD_CHECK', 'true')
# TODO(metzman): Use better system for interpreting env vars. What if env
# var is set to '0'?
diff --git a/infra/cifuzz/config_utils_test.py b/infra/cifuzz/config_utils_test.py
index 71e7450fa..6f87bd4c5 100644
--- a/infra/cifuzz/config_utils_test.py
+++ b/infra/cifuzz/config_utils_test.py
@@ -13,19 +13,38 @@
# limitations under the License.
"""Module for getting the configuration CIFuzz needs to run."""
import os
-import sys
import unittest
import config_utils
-
-# pylint: disable=wrong-import-position,import-error
-sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
import test_helpers
# pylint: disable=no-self-use
+class BaseConfigTest(unittest.TestCase):
+ """Tests for BaseConfig."""
+
+ def setUp(self):
+ test_helpers.patch_environ(self)
+
+ def _create_config(self):
+ return config_utils.BuildFuzzersConfig()
+
+ def test_language_default(self):
+ """Tests that the correct default language is set."""
+ os.environ['BUILD_INTEGRATION_PATH'] = '/path'
+ config = self._create_config()
+ self.assertEqual(config.language, 'c++')
+
+ def test_language(self):
+ """Tests that the correct language is set."""
+ os.environ['BUILD_INTEGRATION_PATH'] = '/path'
+ language = 'python'
+ os.environ['LANGUAGE'] = language
+ config = self._create_config()
+ self.assertEqual(config.language, language)
+
+
class BuildFuzzersConfigTest(unittest.TestCase):
"""Tests for BuildFuzzersConfig."""
diff --git a/infra/cifuzz/coverage.py b/infra/cifuzz/coverage.py
index b5c6fbf1a..9a179c59d 100644
--- a/infra/cifuzz/coverage.py
+++ b/infra/cifuzz/coverage.py
@@ -115,7 +115,7 @@ def _get_latest_cov_report_info(project_name):
LATEST_REPORT_INFO_PATH,
project_name + '.json')
latest_cov_info = get_json_from_url(latest_report_info_url)
- if not latest_cov_info is None:
+ if latest_cov_info is None:
logging.error('Could not get the coverage report json from url: %s.',
latest_report_info_url)
return None
diff --git a/infra/cifuzz/coverage_test.py b/infra/cifuzz/coverage_test.py
index 57120f5f5..1b24d798c 100644
--- a/infra/cifuzz/coverage_test.py
+++ b/infra/cifuzz/coverage_test.py
@@ -21,8 +21,8 @@ import coverage
# pylint: disable=protected-access
-TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'test_files')
+TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ 'test_data')
PROJECT_NAME = 'curl'
REPO_PATH = '/src/curl'
@@ -31,7 +31,7 @@ PROJECT_COV_JSON_FILENAME = 'example_curl_cov.json'
FUZZ_TARGET_COV_JSON_FILENAME = 'example_curl_fuzzer_cov.json'
INVALID_TARGET = 'not-a-fuzz-target'
-with open(os.path.join(TEST_FILES_PATH,
+with open(os.path.join(TEST_DATA_PATH,
PROJECT_COV_JSON_FILENAME),) as cov_file_handle:
PROJECT_COV_INFO = json.loads(cov_file_handle.read())
@@ -39,19 +39,28 @@ with open(os.path.join(TEST_FILES_PATH,
class GetFuzzerStatsDirUrlTest(unittest.TestCase):
"""Tests _get_fuzzer_stats_dir_url."""
- @mock.patch('coverage.get_json_from_url', return_value={})
+ @mock.patch('coverage.get_json_from_url',
+ return_value={
+ 'fuzzer_stats_dir':
+ 'gs://oss-fuzz-coverage/systemd/fuzzer_stats/20210303'
+ })
def test_get_valid_project(self, mocked_get_json_from_url):
"""Tests that a project's coverage report can be downloaded and parsed.
NOTE: This test relies on the PROJECT_NAME repo's coverage report.
The "example" project was not used because it has no coverage reports.
"""
- coverage._get_fuzzer_stats_dir_url(PROJECT_NAME)
+ result = coverage._get_fuzzer_stats_dir_url(PROJECT_NAME)
(url,), _ = mocked_get_json_from_url.call_args
self.assertEqual(
'https://storage.googleapis.com/oss-fuzz-coverage/'
'latest_report_info/curl.json', url)
+ expected_result = (
+ 'https://storage.googleapis.com/oss-fuzz-coverage/systemd/fuzzer_stats/'
+ '20210303')
+ self.assertEqual(result, expected_result)
+
def test_get_invalid_project(self):
"""Tests that passing a bad project returns None."""
self.assertIsNone(coverage._get_fuzzer_stats_dir_url('not-a-proj'))
@@ -98,7 +107,7 @@ class GetFilesCoveredByTargetTest(unittest.TestCase):
def test_valid_target(self):
"""Tests that covered files can be retrieved from a coverage report."""
- with open(os.path.join(TEST_FILES_PATH,
+ with open(os.path.join(TEST_DATA_PATH,
FUZZ_TARGET_COV_JSON_FILENAME),) as file_handle:
fuzzer_cov_info = json.loads(file_handle.read())
@@ -106,7 +115,7 @@ class GetFilesCoveredByTargetTest(unittest.TestCase):
return_value=fuzzer_cov_info):
file_list = self.coverage_getter.get_files_covered_by_target(FUZZ_TARGET)
- curl_files_list_path = os.path.join(TEST_FILES_PATH,
+ curl_files_list_path = os.path.join(TEST_DATA_PATH,
'example_curl_file_list.json')
with open(curl_files_list_path) as file_handle:
expected_file_list = json.loads(file_handle.read())
@@ -152,5 +161,34 @@ class IsFileCoveredTest(unittest.TestCase):
self.assertFalse(coverage.is_file_covered(file_coverage))
+class GetLatestCovReportInfo(unittest.TestCase):
+ """Tests that _get_latest_cov_report_info works as intended."""
+
+ PROJECT = 'project'
+ LATEST_REPORT_INFO_URL = ('https://storage.googleapis.com/oss-fuzz-coverage/'
+ 'latest_report_info/project.json')
+
+ @mock.patch('logging.error')
+ @mock.patch('coverage.get_json_from_url', return_value={'coverage': 1})
+ def test_get_latest_cov_report_info(self, mocked_get_json_from_url,
+ mocked_error):
+ """Tests that _get_latest_cov_report_info works as intended."""
+ result = coverage._get_latest_cov_report_info(self.PROJECT)
+ self.assertEqual(result, {'coverage': 1})
+ mocked_error.assert_not_called()
+ mocked_get_json_from_url.assert_called_with(self.LATEST_REPORT_INFO_URL)
+
+ @mock.patch('logging.error')
+ @mock.patch('coverage.get_json_from_url', return_value=None)
+ def test_get_latest_cov_report_info_fail(self, _, mocked_error):
+ """Tests that _get_latest_cov_report_info works as intended when we can't
+ get latest report info."""
+ result = coverage._get_latest_cov_report_info('project')
+ self.assertIsNone(result)
+ mocked_error.assert_called_with(
+ 'Could not get the coverage report json from url: %s.',
+ self.LATEST_REPORT_INFO_URL)
+
+
if __name__ == '__main__':
unittest.main()
diff --git a/infra/cifuzz/docker.py b/infra/cifuzz/docker.py
new file mode 100644
index 000000000..eb993e28d
--- /dev/null
+++ b/infra/cifuzz/docker.py
@@ -0,0 +1,38 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for dealing with docker."""
+import os
+import sys
+
+# pylint: disable=wrong-import-position,import-error
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import utils
+
+BASE_BUILDER_TAG = 'gcr.io/oss-fuzz-base/base-builder'
+BASE_RUNNER_TAG = 'gcr.io/oss-fuzz-base/base-runner'
+MSAN_LIBS_BUILDER_TAG = 'gcr.io/oss-fuzz-base/msan-libs-builder'
+PROJECT_TAG_PREFIX = 'gcr.io/oss-fuzz/'
+
+
+def get_project_image_name(project):
+ """Returns the name of the project builder image for |project_name|."""
+ return PROJECT_TAG_PREFIX + project
+
+
+def delete_images(images):
+ """Deletes |images|."""
+ command = ['docker', 'rmi', '-f'] + images
+ utils.execute(command)
+ utils.execute(['docker', 'builder', 'prune', '-f'])
diff --git a/infra/cifuzz/environment.py b/infra/cifuzz/environment.py
new file mode 100644
index 000000000..4cc0f846b
--- /dev/null
+++ b/infra/cifuzz/environment.py
@@ -0,0 +1,54 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module for dealing with env vars."""
+
+import ast
+import os
+
+
+def _eval_value(value_string):
+ """Returns evaluated value."""
+ try:
+ return ast.literal_eval(value_string)
+ except: # pylint: disable=bare-except
+ # String fallback.
+ return value_string
+
+
+def get(env_var, default_value=None):
+ """Returns an environment variable value."""
+ value_string = os.getenv(env_var)
+ if value_string is None:
+ return default_value
+
+ return _eval_value(value_string)
+
+
+def get_bool(env_var, default_value=None):
+ """Returns a boolean environment variable value. This is needed because a lot
+ of CIFuzz users specified 'false' for dry-run. So we need to special case
+ this."""
+ value = get(env_var, default_value)
+ if not isinstance(value, str):
+ return bool(value)
+
+ lower_value = value.lower()
+ allowed_values = {'true', 'false'}
+ if lower_value not in allowed_values:
+ raise Exception(('Bool env var {env_var} value {value} is invalid. '
+ 'Must be one of {allowed_values}').format(
+ env_var=env_var,
+ value=value,
+ allowed_values=allowed_values))
+ return lower_value == 'true'
diff --git a/infra/cifuzz/fuzz_target.py b/infra/cifuzz/fuzz_target.py
index 7bccfa4e1..c623bf60d 100644
--- a/infra/cifuzz/fuzz_target.py
+++ b/infra/cifuzz/fuzz_target.py
@@ -16,10 +16,13 @@ import collections
import logging
import os
import re
+import shutil
import stat
import subprocess
import sys
+import docker
+
# pylint: disable=wrong-import-position,import-error
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import utils
@@ -28,6 +31,8 @@ logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
level=logging.DEBUG)
+# Use a fixed seed for determinism. Use len_control=0 since we don't have enough
+# time fuzzing for len_control to make sense (probably).
LIBFUZZER_OPTIONS = '-seed=1337 -len_control=0'
# The number of reproduce attempts for a crash.
@@ -78,6 +83,7 @@ class FuzzTarget:
self.out_dir = out_dir
self.clusterfuzz_deployment = clusterfuzz_deployment
self.config = config
+ self.latest_corpus_path = None
def fuzz(self):
"""Starts the fuzz target run for the length of time specified by duration.
@@ -98,8 +104,7 @@ class FuzzTarget:
command += [
'-e', 'FUZZING_ENGINE=libfuzzer', '-e',
'SANITIZER=' + self.config.sanitizer, '-e', 'CIFUZZ=True', '-e',
- 'RUN_FUZZER_MODE=interactive', 'gcr.io/oss-fuzz-base/base-runner',
- 'bash', '-c'
+ 'RUN_FUZZER_MODE=interactive', docker.BASE_RUNNER_TAG, 'bash', '-c'
]
run_fuzzer_command = 'run_fuzzer {fuzz_target} {options}'.format(
@@ -107,10 +112,10 @@ class FuzzTarget:
options=LIBFUZZER_OPTIONS + ' -max_total_time=' + str(self.duration))
# If corpus can be downloaded use it for fuzzing.
- latest_corpus_path = self.clusterfuzz_deployment.download_corpus(
+ self.latest_corpus_path = self.clusterfuzz_deployment.download_corpus(
self.target_name, self.out_dir)
- if latest_corpus_path:
- run_fuzzer_command = run_fuzzer_command + ' ' + latest_corpus_path
+ if self.latest_corpus_path:
+ run_fuzzer_command = run_fuzzer_command + ' ' + self.latest_corpus_path
command.append(run_fuzzer_command)
logging.info('Running command: %s', ' '.join(command))
@@ -136,10 +141,37 @@ class FuzzTarget:
if not testcase:
logging.error(b'No testcase found in stacktrace: %s.', stderr)
return FuzzResult(None, None)
+
+ utils.binary_print(b'Fuzzer: %s. Detected bug:\n%s' %
+ (self.target_name.encode(), stderr))
if self.is_crash_reportable(testcase):
+ # We found a bug in the fuzz target and we will report it.
return FuzzResult(testcase, stderr)
+
+ # We found a bug but we won't report it.
return FuzzResult(None, None)
+ def free_disk_if_needed(self):
+ """Deletes things that are no longer needed from fuzzing this fuzz target to
+ save disk space if needed."""
+ if not self.config.low_disk_space:
+ return
+ logging.info(
+ 'Deleting corpus, seed corpus and fuzz target of %s to save disk.',
+ self.target_name)
+
+ # Delete the seed corpus, corpus, and fuzz target.
+ if self.latest_corpus_path and os.path.exists(self.latest_corpus_path):
+ # Use ignore_errors=True to fix
+ # https://github.com/google/oss-fuzz/issues/5383.
+ shutil.rmtree(self.latest_corpus_path, ignore_errors=True)
+
+ os.remove(self.target_path)
+ target_seed_corpus_path = self.target_path + '_seed_corpus.zip'
+ if os.path.exists(target_seed_corpus_path):
+ os.remove(target_seed_corpus_path)
+ logging.info('Done deleting.')
+
def is_reproducible(self, testcase, target_path):
"""Checks if the testcase reproduces.
@@ -176,8 +208,7 @@ class FuzzTarget:
]
command += [
- '-t', 'gcr.io/oss-fuzz-base/base-runner', 'reproduce', self.target_name,
- '-runs=100'
+ '-t', docker.BASE_RUNNER_TAG, 'reproduce', self.target_name, '-runs=100'
]
logging.info('Running reproduce command: %s.', ' '.join(command))
@@ -246,7 +277,6 @@ class FuzzTarget:
logging.info('The crash is reproducible. The crash doesn\'t reproduce '
'on old builds. This code change probably introduced the '
'crash.')
-
return True
logging.info('The crash is reproducible on old builds '
diff --git a/infra/cifuzz/fuzz_target_test.py b/infra/cifuzz/fuzz_target_test.py
index 8a506fa59..8bec234dc 100644
--- a/infra/cifuzz/fuzz_target_test.py
+++ b/infra/cifuzz/fuzz_target_test.py
@@ -148,8 +148,7 @@ class GetTestCaseTest(unittest.TestCase):
def test_valid_error_string(self):
"""Tests that get_testcase returns the correct testcase give an error."""
testcase_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'test_files',
- 'example_crash_fuzzer_output.txt')
+ 'test_data', 'example_crash_fuzzer_output.txt')
with open(testcase_path, 'rb') as test_fuzz_output:
parsed_testcase = self.test_target.get_testcase(test_fuzz_output.read())
self.assertEqual(
diff --git a/infra/cifuzz/run_fuzzers.py b/infra/cifuzz/run_fuzzers.py
index 2a2a89e5f..513cfb6fa 100644
--- a/infra/cifuzz/run_fuzzers.py
+++ b/infra/cifuzz/run_fuzzers.py
@@ -91,7 +91,9 @@ class BaseFuzzTargetRunner:
"""Fuzzes with |fuzz_target_obj| and returns the result."""
# TODO(metzman): Make children implement this so that the batch runner can
# do things differently.
- return fuzz_target_obj.fuzz()
+ result = fuzz_target_obj.fuzz()
+ fuzz_target_obj.free_disk_if_needed()
+ return result
@property
def quit_on_bug_found(self):
@@ -100,9 +102,12 @@ class BaseFuzzTargetRunner:
raise NotImplementedError('Child class must implement method')
def get_fuzz_target_artifact(self, target, artifact_name):
- """Returns the path of a fuzzing |artifact| named |artifact_name| for
- |target|."""
- artifact_name = target.target_name + '-' + artifact_name
+ """Returns the path of a fuzzing artifact named |artifact_name| for
+ |fuzz_target|."""
+ artifact_name = '{target_name}-{sanitizer}-{artifact_name}'.format(
+ target_name=target.target_name,
+ sanitizer=self.config.sanitizer,
+ artifact_name=artifact_name)
return os.path.join(self.artifacts_dir, artifact_name)
def create_fuzz_target_obj(self, target_path, run_seconds):
@@ -140,12 +145,9 @@ class BaseFuzzTargetRunner:
target.target_name)
continue
- # We found a bug in the fuzz target.
- utils.binary_print(b'Fuzzer: %s. Detected bug:\n%s' %
- (target.target_name.encode(), result.stacktrace))
-
# TODO(metzman): Do this with filestore.
- testcase_artifact_path = self.get_fuzz_target_artifact(target, 'testcase')
+ testcase_artifact_path = self.get_fuzz_target_artifact(
+ target, os.path.basename(result.testcase))
shutil.move(result.testcase, testcase_artifact_path)
bug_summary_artifact_path = self.get_fuzz_target_artifact(
target, 'bug-summary.txt')
diff --git a/infra/cifuzz/run_fuzzers_entrypoint.py b/infra/cifuzz/run_fuzzers_entrypoint.py
index f810e38f8..46e208dc0 100644
--- a/infra/cifuzz/run_fuzzers_entrypoint.py
+++ b/infra/cifuzz/run_fuzzers_entrypoint.py
@@ -11,11 +11,12 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-"""Runs specific OSS-Fuzz project's fuzzers for CI tools."""
+"""Runs a specific OSS-Fuzz project's fuzzers for CI tools."""
import logging
import sys
import config_utils
+import docker
import run_fuzzers
# pylint: disable=c-extension-no-member
@@ -26,6 +27,21 @@ logging.basicConfig(
level=logging.DEBUG)
+def delete_unneeded_docker_images(config):
+ """Deletes unneeded docker images if running in an environment with low
+ disk space."""
+ if not config.low_disk_space:
+ return
+ logging.info('Deleting builder docker images to save disk space.')
+ project_image = docker.get_project_image_name(config.project_name)
+ images = [
+ project_image,
+ docker.BASE_RUNNER_TAG,
+ docker.MSAN_LIBS_BUILDER_TAG,
+ ]
+ docker.delete_images(images)
+
+
def main():
"""Runs OSS-Fuzz project's fuzzers for CI tools.
This is the entrypoint for the run_fuzzers github action.
@@ -62,6 +78,7 @@ def main():
logging.error('This script needs to be run within Github actions.')
return returncode
+ delete_unneeded_docker_images(config)
# Run the specified project's fuzzers from the build.
result = run_fuzzers.run_fuzzers(config)
if result == run_fuzzers.RunFuzzersResult.ERROR:
diff --git a/infra/cifuzz/run_fuzzers_test.py b/infra/cifuzz/run_fuzzers_test.py
index 847ddf399..b2659903c 100644
--- a/infra/cifuzz/run_fuzzers_test.py
+++ b/infra/cifuzz/run_fuzzers_test.py
@@ -37,13 +37,13 @@ import test_helpers
EXAMPLE_PROJECT = 'example'
# Location of files used for testing.
-TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'test_files')
+TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ 'test_data')
-MEMORY_FUZZER_DIR = os.path.join(TEST_FILES_PATH, 'memory')
+MEMORY_FUZZER_DIR = os.path.join(TEST_DATA_PATH, 'memory')
MEMORY_FUZZER = 'curl_fuzzer_memory'
-UNDEFINED_FUZZER_DIR = os.path.join(TEST_FILES_PATH, 'undefined')
+UNDEFINED_FUZZER_DIR = os.path.join(TEST_DATA_PATH, 'undefined')
UNDEFINED_FUZZER = 'curl_fuzzer_undefined'
FUZZ_SECONDS = 10
@@ -227,7 +227,8 @@ class BaseFuzzTargetRunnerTest(unittest.TestCase):
target.target_name = target_name
fuzz_target_artifact = runner.get_fuzz_target_artifact(
target, artifact_name)
- expected_fuzz_target_artifact = 'artifacts-dir/target_name-artifact-name'
+ expected_fuzz_target_artifact = (
+ 'artifacts-dir/target_name-address-artifact-name')
self.assertEqual(fuzz_target_artifact, expected_fuzz_target_artifact)
@@ -263,7 +264,7 @@ class CiFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase):
magic_mock.target_name = 'target1'
mocked_create_fuzz_target_obj.return_value = magic_mock
self.assertTrue(runner.run_fuzz_targets())
- self.assertIn('target1-testcase', os.listdir(runner.artifacts_dir))
+ self.assertIn('target1-address-testcase', os.listdir(runner.artifacts_dir))
self.assertEqual(mocked_run_fuzz_target.call_count, 1)
@@ -279,7 +280,7 @@ class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase):
def test_run_fuzz_targets_quits(self, mocked_create_fuzz_target_obj,
mocked_run_fuzz_target,
mocked_get_fuzz_targets):
- """Tests that run_fuzz_targets quits on the first crash it finds."""
+ """Tests that run_fuzz_targets doesn't quit on the first crash it finds."""
workspace = 'workspace'
out_path = os.path.join(workspace, 'out')
self.fs.create_dir(out_path)
@@ -290,8 +291,8 @@ class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase):
mocked_get_fuzz_targets.return_value = ['target1', 'target2']
runner.initialize()
- testcase1 = os.path.join(workspace, 'testcase1')
- testcase2 = os.path.join(workspace, 'testcase2')
+ testcase1 = os.path.join(workspace, 'testcase-aaa')
+ testcase2 = os.path.join(workspace, 'testcase-bbb')
self.fs.create_file(testcase1)
self.fs.create_file(testcase2)
stacktrace = b'stacktrace'
@@ -312,7 +313,8 @@ class BatchFuzzTargetRunnerTest(fake_filesystem_unittest.TestCase):
magic_mock.target_name = 'target1'
mocked_create_fuzz_target_obj.return_value = magic_mock
self.assertTrue(runner.run_fuzz_targets())
- self.assertIn('target1-testcase', os.listdir(runner.artifacts_dir))
+ self.assertIn('target1-address-testcase-aaa',
+ os.listdir(runner.artifacts_dir))
self.assertEqual(mocked_run_fuzz_target.call_count, 2)
@@ -333,7 +335,7 @@ class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin,
side_effect=[True, False]):
with tempfile.TemporaryDirectory() as tmp_dir:
workspace = os.path.join(tmp_dir, 'workspace')
- shutil.copytree(TEST_FILES_PATH, workspace)
+ shutil.copytree(TEST_DATA_PATH, workspace)
config = _create_config(fuzz_seconds=FUZZ_SECONDS,
workspace=workspace,
project_name=EXAMPLE_PROJECT)
@@ -349,17 +351,17 @@ class RunAddressFuzzersIntegrationTest(RunFuzzerIntegrationTestMixin,
def test_old_bug_found(self, _):
"""Tests run_fuzzers with a bug found in OSS-Fuzz before."""
config = _create_config(fuzz_seconds=FUZZ_SECONDS,
- workspace=TEST_FILES_PATH,
+ workspace=TEST_DATA_PATH,
project_name=EXAMPLE_PROJECT)
with tempfile.TemporaryDirectory() as tmp_dir:
workspace = os.path.join(tmp_dir, 'workspace')
- shutil.copytree(TEST_FILES_PATH, workspace)
+ shutil.copytree(TEST_DATA_PATH, workspace)
config = _create_config(fuzz_seconds=FUZZ_SECONDS,
- workspace=TEST_FILES_PATH,
+ workspace=TEST_DATA_PATH,
project_name=EXAMPLE_PROJECT)
result = run_fuzzers.run_fuzzers(config)
self.assertEqual(result, run_fuzzers.RunFuzzersResult.NO_BUG_FOUND)
- build_dir = os.path.join(TEST_FILES_PATH, 'out', self.BUILD_DIR_NAME)
+ build_dir = os.path.join(TEST_DATA_PATH, 'out', self.BUILD_DIR_NAME)
self.assertTrue(os.path.exists(build_dir))
self.assertNotEqual(0, len(os.listdir(build_dir)))
diff --git a/infra/cifuzz/stack_parser.py b/infra/cifuzz/stack_parser.py
index 0077caae9..69c44bc2e 100644
--- a/infra/cifuzz/stack_parser.py
+++ b/infra/cifuzz/stack_parser.py
@@ -13,6 +13,8 @@
# limitations under the License.
"""Module for parsing stacks from fuzz targets."""
+import logging
+
# From clusterfuzz: src/python/crash_analysis/crash_analyzer.py
# Used to get the beginning of the stacktrace.
STACKTRACE_TOOL_MARKERS = [
@@ -51,25 +53,33 @@ def parse_fuzzer_output(fuzzer_output, parsed_output_file_path):
parsed_output_file_path: The location to store the parsed output.
"""
# Get index of key file points.
+ begin_stack = None
for marker in STACKTRACE_TOOL_MARKERS:
marker_index = fuzzer_output.find(marker)
- if marker_index:
+ if marker_index != -1:
begin_stack = marker_index
break
- end_stack = -1
+ if begin_stack is None:
+ logging.error(
+ b'Could not find a begin stack marker (%s) in fuzzer output:\n%s',
+ STACKTRACE_TOOL_MARKERS, fuzzer_output)
+ return
+
+ end_stack = None
for marker in STACKTRACE_END_MARKERS:
marker_index = fuzzer_output.find(marker)
- if marker_index:
+ if marker_index != -1:
end_stack = marker_index + len(marker)
break
- if begin_stack is None or end_stack is None:
+ if end_stack is None:
+ logging.error(
+ b'Could not find an end stack marker (%s) in fuzzer output:\n%s',
+ STACKTRACE_END_MARKERS, fuzzer_output)
return
summary_str = fuzzer_output[begin_stack:end_stack]
- if not summary_str:
- return
# Write sections of fuzzer output to specific files.
with open(parsed_output_file_path, 'ab') as summary_handle:
diff --git a/infra/cifuzz/stack_parser_test.py b/infra/cifuzz/stack_parser_test.py
index 9b05710fc..faf601fd5 100644
--- a/infra/cifuzz/stack_parser_test.py
+++ b/infra/cifuzz/stack_parser_test.py
@@ -14,7 +14,9 @@
"""Tests for stack_parser."""
import os
import unittest
+from unittest import mock
+import parameterized
from pyfakefs import fake_filesystem_unittest
import stack_parser
@@ -23,9 +25,9 @@ import stack_parser
# https://github.com/google/oss-fuzz/tree/master/projects/example project.
EXAMPLE_PROJECT = 'example'
-# Location of files used for testing.
-TEST_FILES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'test_files')
+# Location of data used for testing.
+TEST_DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+ 'test_data')
class ParseOutputTest(fake_filesystem_unittest.TestCase):
@@ -33,33 +35,42 @@ class ParseOutputTest(fake_filesystem_unittest.TestCase):
def setUp(self):
self.setUpPyfakefs()
+ self.maxDiff = None # pylint: disable=invalid-name
- def test_parse_valid_output(self):
+ @parameterized.parameterized.expand([('example_crash_fuzzer_output.txt',
+ 'example_crash_fuzzer_bug_summary.txt'),
+ ('msan_crash_fuzzer_output.txt',
+ 'msan_crash_fuzzer_bug_summary.txt')])
+ def test_parse_valid_output(self, fuzzer_output_file, bug_summary_file):
"""Checks that the parse fuzzer output can correctly parse output."""
# Read the fuzzer output from disk.
- fuzzer_output_path = os.path.join(TEST_FILES_PATH,
- 'example_crash_fuzzer_output.txt')
+ fuzzer_output_path = os.path.join(TEST_DATA_PATH, fuzzer_output_file)
self.fs.add_real_file(fuzzer_output_path)
with open(fuzzer_output_path, 'rb') as fuzzer_output_handle:
fuzzer_output = fuzzer_output_handle.read()
bug_summary_path = '/bug-summary.txt'
- stack_parser.parse_fuzzer_output(fuzzer_output, bug_summary_path)
+ with mock.patch('logging.info') as mocked_info:
+ stack_parser.parse_fuzzer_output(fuzzer_output, bug_summary_path)
+ mocked_info.assert_not_called()
+
with open(bug_summary_path) as bug_summary_handle:
bug_summary = bug_summary_handle.read()
# Compare the bug to the expected one.
- expected_bug_summary_path = os.path.join(TEST_FILES_PATH,
- 'bug_summary_example.txt')
+ expected_bug_summary_path = os.path.join(TEST_DATA_PATH, bug_summary_file)
self.fs.add_real_file(expected_bug_summary_path)
with open(expected_bug_summary_path) as expected_bug_summary_handle:
expected_bug_summary = expected_bug_summary_handle.read()
+
self.assertEqual(expected_bug_summary, bug_summary)
def test_parse_invalid_output(self):
"""Checks that no files are created when an invalid input was given."""
artifact_path = '/bug-summary.txt'
- stack_parser.parse_fuzzer_output(b'not a valid output_string',
- artifact_path)
+ with mock.patch('logging.error') as mocked_error:
+ stack_parser.parse_fuzzer_output(b'not a valid output_string',
+ artifact_path)
+ assert mocked_error.call_count
self.assertFalse(os.path.exists(artifact_path))
diff --git a/infra/cifuzz/test_files/bug_summary_example.txt b/infra/cifuzz/test_data/example_crash_fuzzer_bug_summary.txt
index 8caebad0c..8caebad0c 100644
--- a/infra/cifuzz/test_files/bug_summary_example.txt
+++ b/infra/cifuzz/test_data/example_crash_fuzzer_bug_summary.txt
diff --git a/infra/cifuzz/test_files/example_crash_fuzzer_output.txt b/infra/cifuzz/test_data/example_crash_fuzzer_output.txt
index d316f5f40..d316f5f40 100644
--- a/infra/cifuzz/test_files/example_crash_fuzzer_output.txt
+++ b/infra/cifuzz/test_data/example_crash_fuzzer_output.txt
diff --git a/infra/cifuzz/test_files/example_curl_cov.json b/infra/cifuzz/test_data/example_curl_cov.json
index 0936102fd..0936102fd 100644
--- a/infra/cifuzz/test_files/example_curl_cov.json
+++ b/infra/cifuzz/test_data/example_curl_cov.json
diff --git a/infra/cifuzz/test_files/example_curl_file_list.json b/infra/cifuzz/test_data/example_curl_file_list.json
index 0ed1965c5..0ed1965c5 100644
--- a/infra/cifuzz/test_files/example_curl_file_list.json
+++ b/infra/cifuzz/test_data/example_curl_file_list.json
diff --git a/infra/cifuzz/test_files/example_curl_fuzzer_cov.json b/infra/cifuzz/test_data/example_curl_fuzzer_cov.json
index 6f8c2498c..6f8c2498c 100644
--- a/infra/cifuzz/test_files/example_curl_fuzzer_cov.json
+++ b/infra/cifuzz/test_data/example_curl_fuzzer_cov.json
diff --git a/infra/cifuzz/test_files/external-project/Makefile b/infra/cifuzz/test_data/external-project/Makefile
index 2c1773776..2c1773776 100644
--- a/infra/cifuzz/test_files/external-project/Makefile
+++ b/infra/cifuzz/test_data/external-project/Makefile
diff --git a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.cpp b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.cpp
index 71fa8cae2..71fa8cae2 100644
--- a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.cpp
+++ b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.cpp
diff --git a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.dict b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.dict
index 224679bf4..224679bf4 100644
--- a/infra/cifuzz/test_files/external-project/do_stuff_fuzzer.dict
+++ b/infra/cifuzz/test_data/external-project/do_stuff_fuzzer.dict
diff --git a/infra/cifuzz/test_files/external-project/my_api.cpp b/infra/cifuzz/test_data/external-project/my_api.cpp
index 9a2c1bc1c..9a2c1bc1c 100644
--- a/infra/cifuzz/test_files/external-project/my_api.cpp
+++ b/infra/cifuzz/test_data/external-project/my_api.cpp
diff --git a/infra/cifuzz/test_files/external-project/my_api.h b/infra/cifuzz/test_data/external-project/my_api.h
index 325aa15cc..325aa15cc 100644
--- a/infra/cifuzz/test_files/external-project/my_api.h
+++ b/infra/cifuzz/test_data/external-project/my_api.h
diff --git a/infra/cifuzz/test_files/external-project/oss-fuzz/Dockerfile b/infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile
index e9dc33031..e9dc33031 100644
--- a/infra/cifuzz/test_files/external-project/oss-fuzz/Dockerfile
+++ b/infra/cifuzz/test_data/external-project/oss-fuzz/Dockerfile
diff --git a/infra/cifuzz/test_files/external-project/oss-fuzz/build.sh b/infra/cifuzz/test_data/external-project/oss-fuzz/build.sh
index 2c52ef90f..2c52ef90f 100644
--- a/infra/cifuzz/test_files/external-project/oss-fuzz/build.sh
+++ b/infra/cifuzz/test_data/external-project/oss-fuzz/build.sh
diff --git a/infra/cifuzz/test_files/external-project/standalone_fuzz_target_runner.cpp b/infra/cifuzz/test_data/external-project/standalone_fuzz_target_runner.cpp
index 38a0454f0..38a0454f0 100644
--- a/infra/cifuzz/test_files/external-project/standalone_fuzz_target_runner.cpp
+++ b/infra/cifuzz/test_data/external-project/standalone_fuzz_target_runner.cpp
diff --git a/infra/cifuzz/test_files/memory/out/curl_fuzzer_memory b/infra/cifuzz/test_data/memory/out/curl_fuzzer_memory
index c602ce970..c602ce970 100755
--- a/infra/cifuzz/test_files/memory/out/curl_fuzzer_memory
+++ b/infra/cifuzz/test_data/memory/out/curl_fuzzer_memory
Binary files differ
diff --git a/infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt b/infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt
new file mode 100644
index 000000000..b55e9c6b7
--- /dev/null
+++ b/infra/cifuzz/test_data/msan_crash_fuzzer_bug_summary.txt
@@ -0,0 +1,22 @@
+MemorySanitizer: use-of-uninitialized-value
+#0 0x52675f in LLVMFuzzerTestOneInput /src/cifuzz-example/do_stuff_fuzzer.cpp:13:7
+#1 0x45a431 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:599:15
+#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3
+#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3
+#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6
+#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10
+#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f)
+#7 0x41eab8 in _start (out/do_stuff_fuzzer+0x41eab8)
+
+DEDUP_TOKEN: LLVMFuzzerTestOneInput--fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&)
+Uninitialized value was created by a heap allocation
+#0 0x4d57ad in malloc /src/llvm-project/compiler-rt/lib/msan/msan_interceptors.cpp:901:3
+#1 0x437c07 in operator new(unsigned long) (out/do_stuff_fuzzer+0x437c07)
+#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3
+#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3
+#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6
+#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10
+#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f)
+DEDUP_TOKEN: malloc--operator new(unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&)
+
+SUMMARY: \ No newline at end of file
diff --git a/infra/cifuzz/test_data/msan_crash_fuzzer_output.txt b/infra/cifuzz/test_data/msan_crash_fuzzer_output.txt
new file mode 100644
index 000000000..c803bfb1c
--- /dev/null
+++ b/infra/cifuzz/test_data/msan_crash_fuzzer_output.txt
@@ -0,0 +1,39 @@
+Dictionary: 3 entries
+INFO: Running with entropic power schedule (0xFF, 100).
+INFO: Seed: 1337
+INFO: Loaded 1 modules (184 inline 8-bit counters): 184 [0x829300, 0x8293b8),
+INFO: Loaded 1 PC tables (184 PCs): 184 [0x5dc910,0x5dd490),
+INFO: 5 files found in /tmp/do_stuff_fuzzer_corpus
+INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes
+==13==WARNING: MemorySanitizer: use-of-uninitialized-value
+#0 0x52675f in LLVMFuzzerTestOneInput /src/cifuzz-example/do_stuff_fuzzer.cpp:13:7
+#1 0x45a431 in fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:599:15
+#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3
+#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3
+#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6
+#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10
+#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f)
+#7 0x41eab8 in _start (out/do_stuff_fuzzer+0x41eab8)
+
+DEDUP_TOKEN: LLVMFuzzerTestOneInput--fuzzer::Fuzzer::ExecuteCallback(unsigned char const*, unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&)
+Uninitialized value was created by a heap allocation
+#0 0x4d57ad in malloc /src/llvm-project/compiler-rt/lib/msan/msan_interceptors.cpp:901:3
+#1 0x437c07 in operator new(unsigned long) (out/do_stuff_fuzzer+0x437c07)
+#2 0x45ba46 in fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:792:3
+#3 0x45bed9 in fuzzer::Fuzzer::Loop(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerLoop.cpp:845:3
+#4 0x44a4bc in fuzzer::FuzzerDriver(int*, char***, int (*)(unsigned char const*, unsigned long)) /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerDriver.cpp:906:6
+#5 0x474432 in main /src/llvm-project/compiler-rt/lib/fuzzer/FuzzerMain.cpp:20:10
+#6 0x7eff5562683f in __libc_start_main (/lib/x86_64-linux-gnu/libc.so.6+0x2083f)
+DEDUP_TOKEN: malloc--operator new(unsigned long)--fuzzer::Fuzzer::ReadAndExecuteSeedCorpora(std::__Fuzzer::vector<fuzzer::SizedFile, fuzzer::fuzzer_allocator<fuzzer::SizedFile> >&)
+
+SUMMARY: MemorySanitizer: use-of-uninitialized-value /src/cifuzz-example/do_stuff_fuzzer.cpp:13:7 in LLVMFuzzerTestOneInput
+Unique heap origins: 65
+Stack depot allocated bytes: 4424
+Unique origin histories: 29
+History depot allocated bytes: 696
+Exiting
+MS: 0 ; base unit: 0000000000000000000000000000000000000000
+
+
+artifact_prefix='./'; Test unit written to ./crash-da39a3ee5e6b4b0d3255bfef95601890afd80709
+Base64:
diff --git a/infra/cifuzz/test_files/out/example_crash_fuzzer b/infra/cifuzz/test_data/out/example_crash_fuzzer
index 704800dda..704800dda 100755
--- a/infra/cifuzz/test_files/out/example_crash_fuzzer
+++ b/infra/cifuzz/test_data/out/example_crash_fuzzer
Binary files differ
diff --git a/infra/cifuzz/test_files/out/example_nocrash_fuzzer b/infra/cifuzz/test_data/out/example_nocrash_fuzzer
index e4ff86042..e4ff86042 100755
--- a/infra/cifuzz/test_files/out/example_nocrash_fuzzer
+++ b/infra/cifuzz/test_data/out/example_nocrash_fuzzer
Binary files differ
diff --git a/infra/cifuzz/test_files/undefined/out/curl_fuzzer_undefined b/infra/cifuzz/test_data/undefined/out/curl_fuzzer_undefined
index 504cab108..504cab108 100755
--- a/infra/cifuzz/test_files/undefined/out/curl_fuzzer_undefined
+++ b/infra/cifuzz/test_data/undefined/out/curl_fuzzer_undefined
Binary files differ
diff --git a/infra/go/coverage/gocovsum/gocovsum.go b/infra/go/coverage/gocovsum/gocovsum.go
deleted file mode 100644
index 206600619..000000000
--- a/infra/go/coverage/gocovsum/gocovsum.go
+++ /dev/null
@@ -1,126 +0,0 @@
-package main
-
-import (
- "encoding/json"
- "flag"
- "fmt"
- "log"
-
- "go/ast"
- "go/parser"
- "go/token"
- "os"
- "path"
-
- "golang.org/x/tools/cover"
-)
-
-type CoverageTotal struct {
- Count int `json:"count"`
- Covered int `json:"covered"`
- Uncovered int `json:"notcovered"`
- Percent float64 `json:"percent"`
-}
-
-type CoverageTotals struct {
- Functions CoverageTotal `json:"functions,omitempty"`
- Lines CoverageTotal `json:"lines,omitempty"`
- Regions CoverageTotal `json:"regions,omitempty"`
-}
-
-type CoverageData struct {
- Totals CoverageTotals `json:"totals,omitempty"`
-}
-
-type PositionInterval struct {
- start token.Position
- end token.Position
-}
-
-type CoverageSummary struct {
- Data []CoverageData `json:"data,omitempty"`
- Type string `json:"type,omitempty"`
- Version string `json:"version,omitempty"`
-}
-
-func isFunctionCovered(s token.Position, e token.Position, blocks []cover.ProfileBlock) bool {
- for _, b := range blocks {
- if b.StartLine >= s.Line && b.StartLine <= e.Line && b.EndLine >= s.Line && b.EndLine <= e.Line {
- if b.Count > 0 {
- return true
- }
- }
- }
- return false
-}
-
-func main() {
- flag.Parse()
-
- if len(flag.Args()) != 1 {
- log.Fatalf("needs exactly one argument")
- }
- profiles, err := cover.ParseProfiles(flag.Args()[0])
- if err != nil {
- log.Fatalf("failed to parse profiles: %v", err)
- }
- r := CoverageSummary{}
- r.Type = "oss-fuzz.go.coverage.json.export"
- r.Version = "1.0.0"
- r.Data = make([]CoverageData, 1)
- gopath := os.Getenv("GOPATH")
- if len(gopath) == 0 {
- gopath = os.Getenv("HOME") + "/go"
- }
- for _, p := range profiles {
- fset := token.NewFileSet() // positions are relative to fset
- f, err := parser.ParseFile(fset, path.Join(gopath, "src", p.FileName), nil, 0)
- if err != nil {
- panic(err)
- }
- ast.Inspect(f, func(n ast.Node) bool {
- switch x := n.(type) {
- case *ast.FuncLit:
- startf := fset.Position(x.Pos())
- endf := fset.Position(x.End())
- r.Data[0].Totals.Functions.Count++
- if isFunctionCovered(startf, endf, p.Blocks) {
- r.Data[0].Totals.Functions.Covered++
- } else {
- r.Data[0].Totals.Functions.Uncovered++
- }
- case *ast.FuncDecl:
- startf := fset.Position(x.Pos())
- endf := fset.Position(x.End())
- r.Data[0].Totals.Functions.Count++
- if isFunctionCovered(startf, endf, p.Blocks) {
- r.Data[0].Totals.Functions.Covered++
- } else {
- r.Data[0].Totals.Functions.Uncovered++
- }
- }
- return true
- })
-
- for _, b := range p.Blocks {
- r.Data[0].Totals.Regions.Count++
- if b.Count > 0 {
- r.Data[0].Totals.Regions.Covered++
- } else {
- r.Data[0].Totals.Regions.Uncovered++
- }
-
- r.Data[0].Totals.Lines.Count += b.NumStmt
- if b.Count > 0 {
- r.Data[0].Totals.Lines.Covered += b.NumStmt
- } else {
- r.Data[0].Totals.Lines.Uncovered += b.NumStmt
- }
- }
- }
- r.Data[0].Totals.Regions.Percent = float64(100*r.Data[0].Totals.Regions.Covered) / float64(r.Data[0].Totals.Regions.Count)
- r.Data[0].Totals.Lines.Percent = float64(100*r.Data[0].Totals.Lines.Covered) / float64(r.Data[0].Totals.Lines.Count)
- r.Data[0].Totals.Functions.Percent = float64(100*r.Data[0].Totals.Functions.Covered) / float64(r.Data[0].Totals.Functions.Count)
- o, _ := json.Marshal(r)
- fmt.Printf(string(o))
-}
diff --git a/infra/go/coverage/pprof-merge/go.mod b/infra/go/coverage/pprof-merge/go.mod
deleted file mode 100644
index 5d5b514ac..000000000
--- a/infra/go/coverage/pprof-merge/go.mod
+++ /dev/null
@@ -1,5 +0,0 @@
-module github.com/rakyll/pprof-merge
-
-go 1.13
-
-require github.com/google/pprof v0.0.0-20190908185732-236ed259b199
diff --git a/infra/helper.py b/infra/helper.py
index 8a0a640c1..e24df4ded 100755
--- a/infra/helper.py
+++ b/infra/helper.py
@@ -22,7 +22,6 @@ from multiprocessing.dummy import Pool as ThreadPool
import argparse
import datetime
import errno
-import multiprocessing
import os
import pipes
import re
@@ -59,15 +58,61 @@ CORPUS_BACKUP_URL_FORMAT = (
PROJECT_LANGUAGE_REGEX = re.compile(r'\s*language\s*:\s*([^\s]+)')
# Languages from project.yaml that have code coverage support.
-LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go']
+LANGUAGES_WITH_COVERAGE_SUPPORT = ['c', 'c++', 'go', 'rust']
+# pylint: disable=too-many-lines
-def main(): # pylint: disable=too-many-branches,too-many-return-statements,too-many-statements
+
+def main(): # pylint: disable=too-many-branches,too-many-return-statements
"""Get subcommand from program arguments and do it."""
os.chdir(OSS_FUZZ_DIR)
if not os.path.exists(BUILD_DIR):
os.mkdir(BUILD_DIR)
+ args = parse_args()
+
+ # We have different default values for `sanitizer` depending on the `engine`.
+ # Some commands do not have `sanitizer` argument, so `hasattr` is necessary.
+ if hasattr(args, 'sanitizer') and not args.sanitizer:
+ if args.engine == 'dataflow':
+ args.sanitizer = 'dataflow'
+ else:
+ args.sanitizer = 'address'
+
+ if args.command == 'generate':
+ return generate(args)
+ if args.command == 'build_image':
+ return build_image(args)
+ if args.command == 'build_fuzzers':
+ return build_fuzzers(args)
+ if args.command == 'check_build':
+ return check_build(args)
+ if args.command == 'download_corpora':
+ return download_corpora(args)
+ if args.command == 'run_fuzzer':
+ return run_fuzzer(args)
+ if args.command == 'coverage':
+ return coverage(args)
+ if args.command == 'reproduce':
+ return reproduce(args)
+ if args.command == 'shell':
+ return shell(args)
+ if args.command == 'pull_images':
+ return pull_images(args)
+
+ return 0
+
+
+def parse_args(args=None):
+ """Parses args using argparser and returns parsed args."""
+ # Use default argument None for args so that in production, argparse does its
+ # normal behavior, but unittesting is easier.
+ parser = get_parser()
+ return parser.parse_args(args)
+
+
+def get_parser(): # pylint: disable=too-many-statements
+ """Returns an argparse parser."""
parser = argparse.ArgumentParser('helper.py', description='oss-fuzz helpers')
subparsers = parser.add_subparsers(dest='command')
@@ -112,8 +157,9 @@ def main(): # pylint: disable=too-many-branches,too-many-return-statements,too-
_add_engine_args(
check_build_parser,
choices=['libfuzzer', 'afl', 'honggfuzz', 'dataflow', 'none'])
- _add_sanitizer_args(check_build_parser,
- choices=['address', 'memory', 'undefined', 'dataflow'])
+ _add_sanitizer_args(
+ check_build_parser,
+ choices=['address', 'memory', 'undefined', 'dataflow', 'thread'])
_add_environment_args(check_build_parser)
check_build_parser.add_argument('project_name', help='name of the project')
check_build_parser.add_argument('fuzzer_name',
@@ -189,39 +235,7 @@ def main(): # pylint: disable=too-many-branches,too-many-return-statements,too-
_add_environment_args(shell_parser)
subparsers.add_parser('pull_images', help='Pull base images.')
-
- args = parser.parse_args()
-
- # We have different default values for `sanitizer` depending on the `engine`.
- # Some commands do not have `sanitizer` argument, so `hasattr` is necessary.
- if hasattr(args, 'sanitizer') and not args.sanitizer:
- if args.engine == 'dataflow':
- args.sanitizer = 'dataflow'
- else:
- args.sanitizer = 'address'
-
- if args.command == 'generate':
- return generate(args)
- if args.command == 'build_image':
- return build_image(args)
- if args.command == 'build_fuzzers':
- return build_fuzzers(args)
- if args.command == 'check_build':
- return check_build(args)
- if args.command == 'download_corpora':
- return download_corpora(args)
- if args.command == 'run_fuzzer':
- return run_fuzzer(args)
- if args.command == 'coverage':
- return coverage(args)
- if args.command == 'reproduce':
- return reproduce(args)
- if args.command == 'shell':
- return shell(args)
- if args.command == 'pull_images':
- return pull_images(args)
-
- return 0
+ return parser
def is_base_image(image_name):
@@ -335,7 +349,7 @@ def _add_engine_args(parser,
def _add_sanitizer_args(parser,
choices=('address', 'memory', 'undefined', 'coverage',
- 'dataflow')):
+ 'dataflow', 'thread')):
"""Add common sanitizer args."""
parser.add_argument(
'--sanitizer',
@@ -632,7 +646,7 @@ def check_build(args):
]
if args.fuzzer_name:
- run_args += ['test_one', os.path.join('/out', args.fuzzer_name)]
+ run_args += ['test_one.py', args.fuzzer_name]
else:
run_args.append('test_all.py')
@@ -672,14 +686,14 @@ def _get_latest_corpus(project_name, fuzz_target, base_corpus_dir):
fuzz_target=fuzz_target)
command = ['gsutil', 'ls', corpus_backup_url]
- corpus_listing = subprocess.Popen(command,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
- output, error = corpus_listing.communicate()
+ # Don't capture stderr. We want it to print in real time, in case gsutil is
+ # asking for two-factor authentication.
+ corpus_listing = subprocess.Popen(command, stdout=subprocess.PIPE)
+ output, _ = corpus_listing.communicate()
# Some fuzz targets (e.g. new ones) may not have corpus yet, just skip those.
if corpus_listing.returncode:
- print('WARNING: corpus for {0} not found:\n{1}'.format(fuzz_target, error),
+ print('WARNING: corpus for {0} not found:\n'.format(fuzz_target),
file=sys.stderr)
return
@@ -736,7 +750,7 @@ def download_corpora(args):
print('Downloading corpora for %s project to %s' %
(args.project_name, corpus_dir))
- thread_pool = ThreadPool(multiprocessing.cpu_count())
+ thread_pool = ThreadPool()
return all(thread_pool.map(_download_for_single_target, fuzz_targets))
@@ -956,9 +970,11 @@ def shell(args):
'FUZZING_ENGINE=' + args.engine,
'SANITIZER=' + args.sanitizer,
'ARCHITECTURE=' + args.architecture,
- 'FUZZING_LANGUAGE=' + _get_project_language(args.project_name),
]
+ if args.project_name != 'base-runner-debug':
+ env.append('FUZZING_LANGUAGE=' + _get_project_language(args.project_name))
+
if args.e:
env += args.e
diff --git a/infra/helper_test.py b/infra/helper_test.py
new file mode 100644
index 000000000..d899a835b
--- /dev/null
+++ b/infra/helper_test.py
@@ -0,0 +1,35 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for helper.py"""
+
+import unittest
+from unittest import mock
+
+import helper
+
+
+class TestShell(unittest.TestCase):
+ """Tests 'shell' command."""
+
+ @mock.patch('helper.docker_run')
+ @mock.patch('helper.build_image_impl')
+ def test_base_runner_debug(self, mocked_build_image_impl, _):
+ """Tests that shell base-runner-debug works as intended."""
+ image_name = 'base-runner-debug'
+ unparsed_args = ['shell', image_name]
+ args = helper.parse_args(unparsed_args)
+ args.sanitizer = 'address'
+ result = helper.shell(args)
+ mocked_build_image_impl.assert_called_with(image_name)
+ self.assertEqual(result, 0)
diff --git a/infra/presubmit.py b/infra/presubmit.py
index bd5e9c327..90b4f90ac 100755
--- a/infra/presubmit.py
+++ b/infra/presubmit.py
@@ -104,6 +104,7 @@ class ProjectYamlChecker:
'c',
'c++',
'go',
+ 'jvm',
'python',
'rust',
]
@@ -381,8 +382,9 @@ def run_nonbuild_tests(parallel):
def run_tests(_=None, parallel=False):
"""Runs all unit tests."""
- success = run_nonbuild_tests(parallel)
- return success and run_build_tests()
+ nonbuild_success = run_nonbuild_tests(parallel)
+ build_success = run_build_tests()
+ return nonbuild_success and build_success
def get_all_files():
diff --git a/infra/repo_manager.py b/infra/repo_manager.py
index a5781b89a..a0b97b3ef 100644
--- a/infra/repo_manager.py
+++ b/infra/repo_manager.py
@@ -127,6 +127,14 @@ class RepoManager:
return out.strip()
+ def fetch_all_remotes(self):
+ """Fetch all remotes for checkouts that track a single branch."""
+ self.git([
+ 'config', 'remote.origin.fetch', '+refs/heads/*:refs/remotes/origin/*'
+ ],
+ check_result=True)
+ self.git(['remote', 'update'], check_result=True)
+
def get_commit_list(self, newest_commit, oldest_commit=None):
"""Gets the list of commits(inclusive) between the old and new commits.
diff --git a/infra/testcases/curl_test_data b/infra/testcases/curl_test_data
deleted file mode 100644
index ed4b54ea3..000000000
--- a/infra/testcases/curl_test_data
+++ /dev/null
Binary files differ
diff --git a/infra/testcases/libarchive_test_data b/infra/testcases/libarchive_test_data
deleted file mode 100644
index 928bfec97..000000000
--- a/infra/testcases/libarchive_test_data
+++ /dev/null
Binary files differ
diff --git a/infra/testcases/ndpi_test_data b/infra/testcases/ndpi_test_data
deleted file mode 100644
index 010af8604..000000000
--- a/infra/testcases/ndpi_test_data
+++ /dev/null
Binary files differ
diff --git a/infra/testcases/usrsctp_test_data b/infra/testcases/usrsctp_test_data
deleted file mode 100644
index fa90322a2..000000000
--- a/infra/testcases/usrsctp_test_data
+++ /dev/null
Binary files differ
diff --git a/infra/testcases/yara_test_data b/infra/testcases/yara_test_data
deleted file mode 100644
index e2a0b94af..000000000
--- a/infra/testcases/yara_test_data
+++ /dev/null
@@ -1 +0,0 @@
-rule N{condition:for 1r in r(r \ No newline at end of file
diff --git a/infra/utils_test.py b/infra/utils_test.py
index a56295c93..aa6ec7ba7 100644
--- a/infra/utils_test.py
+++ b/infra/utils_test.py
@@ -24,7 +24,7 @@ import helper
EXAMPLE_PROJECT = 'example'
TEST_OUT_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
- 'cifuzz', 'test_files', 'out')
+ 'cifuzz', 'test_data', 'out')
class IsFuzzTargetLocalTest(unittest.TestCase):