diff options
Diffstat (limited to 'infra/base-images')
34 files changed, 1227 insertions, 293 deletions
diff --git a/infra/base-images/Jenkinsfile b/infra/base-images/Jenkinsfile deleted file mode 100644 index 67b22e9b0..000000000 --- a/infra/base-images/Jenkinsfile +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2016 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//////////////////////////////////////////////////////////////////////////////// - -// Jenkins build script for base images. -node { - git url: 'https://github.com/google/oss-fuzz/' - - stage("infra/base-images/all.sh") { - sh "infra/base-images/all.sh --no-cache" - } - - stage("docker push") { - def images = ['ossfuzz/base-image', 'ossfuzz/base-clang', 'ossfuzz/base-libfuzzer', - 'ossfuzz/base-runner', 'ossfuzz/base-runner-debug', - 'ossfuzz/base-builder',] - - docker.withRegistry('', 'docker-login') { - for (int i = 0; i < images.size(); i++) { - def image = images[i] - docker.image(image).push() - } - } - } -} diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index 6f596d5ba..d802f247a 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -29,6 +29,7 @@ RUN dpkg --add-architecture i386 && \ jq \ libc6-dev-i386 \ patchelf \ + rsync \ subversion \ zip @@ -57,12 +58,14 @@ RUN export PYTHON_DEPS="\ ln -s /usr/bin/python3 /usr/bin/python && \ cd .. && \ rm -r /tmp/Python-$PYTHON_VERSION.tar.xz /tmp/Python-$PYTHON_VERSION && \ - apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888 + rm -rf /usr/local/lib/python3.8/test && \ + apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888 # Install latest atheris for python fuzzing, pyinstaller for fuzzer packaging, # six for Bazel rules. -RUN unset CFLAGS CXXFLAGS && pip3 install -v \ - atheris pyinstaller==4.1 six==1.15.0 +RUN unset CFLAGS CXXFLAGS && pip3 install -v --no-cache-dir \ + atheris pyinstaller==4.1 six==1.15.0 && \ + rm -rf /tmp/* # Download and install the latest stable Go. RUN cd /tmp && \ @@ -87,15 +90,42 @@ ENV CARGO_HOME=/rust ENV RUSTUP_HOME=/rust/rustup ENV PATH=$PATH:/rust/bin RUN curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal -RUN cargo install cargo-fuzz +RUN cargo install cargo-fuzz && rm -rf /rust/registry # Needed to recompile rust std library for MSAN RUN rustup component add rust-src --toolchain nightly +# Set up custom environment variable for source code copy for coverage reports +ENV OSSFUZZ_RUSTPATH /rust # Install Bazel through Bazelisk, which automatically fetches the latest Bazel version. ENV BAZELISK_VERSION 1.7.4 RUN curl -L https://github.com/bazelbuild/bazelisk/releases/download/v$BAZELISK_VERSION/bazelisk-linux-amd64 -o /usr/local/bin/bazel && \ chmod +x /usr/local/bin/bazel +# Install OpenJDK 15 and trim its size by removing unused components. +ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64 +ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server +ENV PATH=$PATH:$JAVA_HOME/bin +RUN cd /tmp && \ + curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \ + mkdir -p $JAVA_HOME && \ + tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ + rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \ + rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip + +# Install the latest Jazzer in $OUT. +# jazzer_api_deploy.jar is required only at build-time, the agent and the +# drivers are copied to $OUT as they need to be present on the runners. +ENV JAZZER_API_PATH "/usr/local/lib/jazzer_api_deploy.jar" +RUN cd $SRC/ && \ + git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \ + cd jazzer && \ + bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \ + //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //agent:jazzer_api_deploy.jar && \ + cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan /usr/local/bin/ && \ + cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH && \ + rm -rf ~/.cache/bazel ~/.cache/bazelisk && \ + rm -rf $SRC/jazzer + # Default build flags for various sanitizers. ENV SANITIZER_FLAGS_address "-fsanitize=address -fsanitize-address-use-after-scope" @@ -106,6 +136,8 @@ ENV SANITIZER_FLAGS_memory "-fsanitize=memory -fsanitize-memory-track-origins" ENV SANITIZER_FLAGS_dataflow "-fsanitize=dataflow" +ENV SANITIZER_FLAGS_thread "-fsanitize=thread" + # Do not use any sanitizers in the coverage build. ENV SANITIZER_FLAGS_coverage "" @@ -144,15 +176,12 @@ ENV LIB_FUZZING_ENGINE="/usr/lib/libFuzzingEngine.a" # TODO: remove after tpm2 catchup. ENV FUZZER_LDFLAGS "" -ENV PRECOMPILED_DIR="/usr/lib/precompiled" -RUN mkdir $PRECOMPILED_DIR - WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout aeb7d7048371cd91ab9280c3958f1c35e5d5e758 + git checkout 2102264acf5c271b7560a82771b3af8136af9354 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ @@ -161,12 +190,14 @@ RUN cd $SRC && \ tar -xzv --strip-components=1 -f $SRC/oss-fuzz.tar.gz && \ rm -rf examples $SRC/oss-fuzz.tar.gz -COPY compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ - compile_go_fuzzer precompile_honggfuzz srcmap write_labels.py /usr/local/bin/ +COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ + compile_go_fuzzer precompile_honggfuzz precompile_afl debug_afl srcmap \ + write_labels.py bazel_build_fuzz_tests /usr/local/bin/ COPY detect_repo.py /opt/cifuzz/ COPY ossfuzz_coverage_runner.go $GOPATH RUN precompile_honggfuzz +RUN precompile_afl CMD ["compile"] diff --git a/infra/base-images/base-builder/bazel_build_fuzz_tests b/infra/base-images/base-builder/bazel_build_fuzz_tests new file mode 100755 index 000000000..86740ee01 --- /dev/null +++ b/infra/base-images/base-builder/bazel_build_fuzz_tests @@ -0,0 +1,80 @@ +#!/bin/bash -eu +# +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +: "${BAZEL_FUZZ_TEST_TAG:=fuzz-test}" +: "${BAZEL_FUZZ_TEST_EXCLUDE_TAG:=no-oss-fuzz}" +: "${BAZEL_PACKAGE_SUFFIX:=_oss_fuzz}" +: "${BAZEL_TOOL:=bazel}" +: "${BAZEL_EXTRA_BUILD_FLAGS:=}" + +if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then + BAZEL_FUZZ_TEST_QUERY=" + let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in + \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests) + " +fi + +echo "Using Bazel query to find fuzz targets: ${BAZEL_FUZZ_TEST_QUERY}" + +declare -r OSS_FUZZ_TESTS=( + $(bazel query "${BAZEL_FUZZ_TEST_QUERY}" | sed "s/$/${BAZEL_PACKAGE_SUFFIX}/") +) + +echo "Found ${#OSS_FUZZ_TESTS[@]} fuzz test packages:" +for oss_fuzz_test in "${OSS_FUZZ_TESTS[@]}"; do + echo " ${oss_fuzz_test}" +done + +declare -r BAZEL_BUILD_FLAGS=( + "-c" "opt" + "--//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \ + "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \ + "--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \ + "--linkopt=-lc++" \ + "--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \ + ${BAZEL_EXTRA_BUILD_FLAGS[*]} +) + +echo "Building the fuzz tests with the following Bazel options:" +echo " ${BAZEL_BUILD_FLAGS[@]}" + +${BAZEL_TOOL} build "${BAZEL_BUILD_FLAGS[@]}" "${OSS_FUZZ_TESTS[@]}" + +echo "Extracting the fuzz test packages in the output directory." +for oss_fuzz_archive in $(find bazel-bin/ -name "*${BAZEL_PACKAGE_SUFFIX}.tar"); do + tar -xvf "${oss_fuzz_archive}" -C "${OUT}" +done + +if [ "$SANITIZER" = "coverage" ]; then + echo "Collecting the repository source files for coverage tracking." + declare -r COVERAGE_SOURCES="${OUT}/proc/self/cwd" + mkdir -p "${COVERAGE_SOURCES}" + declare -r RSYNC_FILTER_ARGS=( + "--include" "*.h" + "--include" "*.cc" + "--include" "*.hpp" + "--include" "*.cpp" + "--include" "*.c" + "--include" "*.inc" + "--include" "*/" + "--exclude" "*" + ) + rsync -avLk "${RSYNC_FILTER_ARGS[@]}" \ + "$(bazel info execution_root)/" \ + "${COVERAGE_SOURCES}/" +fi diff --git a/infra/base-images/base-builder/cargo b/infra/base-images/base-builder/cargo new file mode 100755 index 000000000..bed8e7660 --- /dev/null +++ b/infra/base-images/base-builder/cargo @@ -0,0 +1,51 @@ +#!/bin/bash -eu +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# This is a wrapper around calling cargo +# This just expands RUSTFLAGS in case of a coverage build +# We need this until https://github.com/rust-lang/cargo/issues/5450 is merged +# because cargo uses relative paths for the current crate +# and absolute paths for its dependencies +# +################################################################################ + +if [ "$SANITIZER" = "coverage" ] && [ $1 = "build" ] +then + crate_src_abspath=`cargo metadata --no-deps --format-version 1 | jq -r '.workspace_root'` + export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix src=$crate_src_abspath/src" +fi + +if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ] +then + # hack to turn cargo fuzz build into cargo build so as to get coverage + # cargo fuzz adds "--target" "x86_64-unknown-linux-gnu" + ( + # go into fuzz directory if not already the case + cd fuzz || true + fuzz_src_abspath=`pwd` + export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix fuzz_targets=$fuzz_src_abspath/fuzz_targets" + # we do not want to trigger debug assertions and stops + export RUSTFLAGS="$RUSTFLAGS -C debug-assertions=no" + # do not optimize with --release, leading to Malformed instrumentation profile data + cargo build --bins + # copies the build output in the expected target directory + cd `cargo metadata --format-version 1 --no-deps | jq -r '.target_directory'` + mkdir -p x86_64-unknown-linux-gnu/release + cp -r debug/* x86_64-unknown-linux-gnu/release/ + ) + exit 0 +fi + +/rust/bin/cargo "$@" diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index 2bf20b1e3..78453c98c 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -22,6 +22,21 @@ if [ "$SANITIZER" = "dataflow" ] && [ "$FUZZING_ENGINE" != "dataflow" ]; then exit 1 fi +if [ "$FUZZING_LANGUAGE" = "jvm" ]; then + if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then + echo "ERROR: JVM projects can be fuzzed with libFuzzer engine only." + exit 1 + fi + if [ "$SANITIZER" != "address" ]; then + echo "ERROR: JVM projects can be fuzzed with AddressSanitizer only." + exit 1 + fi + if [ "$ARCHITECTURE" != "x86_64" ]; then + echo "ERROR: JVM projects can be fuzzed on x86_64 architecture only." + exit 1 + fi +fi + if [ "$FUZZING_LANGUAGE" = "python" ]; then if [ "$FUZZING_ENGINE" != "libfuzzer" ]; then echo "ERROR: Python projects can be fuzzed with libFuzzer engine only." @@ -46,7 +61,8 @@ if [[ $ARCHITECTURE == "i386" ]]; then export CFLAGS="-m32 $CFLAGS" cp -R /usr/i386/lib/* /usr/lib fi -if [[ $FUZZING_ENGINE != "none" ]]; then +# JVM projects are fuzzed with Jazzer, which has libFuzzer built in. +if [[ $FUZZING_ENGINE != "none" ]] && [[ $FUZZING_LANGUAGE != "jvm" ]]; then # compile script might override environment, use . to call it. . compile_${FUZZING_ENGINE} fi @@ -87,6 +103,11 @@ if [ "$SANITIZER" != "undefined" ] && [ "$SANITIZER" != "coverage" ] && [ "$ARCH else export RUSTFLAGS="--cfg fuzzing -Cdebuginfo=1 -Cforce-frame-pointers" fi +if [ "$SANITIZER" = "coverage" ] +then + # link to C++ from comment in f5098035eb1a14aa966c8651d88ea3d64323823d + export RUSTFLAGS="$RUSTFLAGS -Zinstrument-coverage -C link-arg=-lc++" +fi # Add Rust libfuzzer flags. # See https://github.com/rust-fuzz/libfuzzer/blob/master/build.rs#L12. @@ -113,6 +134,11 @@ fi # Copy latest llvm-symbolizer in $OUT for stack symbolization. cp $(which llvm-symbolizer) $OUT/ +# Copy Jazzer to $OUT if needed. +if [ "$FUZZING_LANGUAGE" = "jvm" ]; then + cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $(which jazzer_driver_asan) $OUT/ +fi + echo "---------------------------------------------------------------" echo "CC=$CC" echo "CXX=$CXX" @@ -124,7 +150,7 @@ BUILD_CMD="bash -eux $SRC/build.sh" # We need to preserve source code files for generating a code coverage report. # We need exact files that were compiled, so copy both $SRC and $WORK dirs. -COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OUT" +COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH $OUT" if [ "${BUILD_UID-0}" -ne "0" ]; then adduser -u $BUILD_UID --disabled-password --gecos '' builder diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl index 318eca44e..dc6624459 100644 --- a/infra/base-images/base-builder/compile_afl +++ b/infra/base-images/base-builder/compile_afl @@ -15,43 +15,78 @@ # ################################################################################ -echo "Compiling afl++" +# afl++ configuration options. +# The 'env|grep' setup ensures we do not trigger the linter. +# The variables need to be set to "1" here - or before running this script. -# Build and copy afl++ tools necessary for fuzzing. +# AFL++ settings. +export AFL_LLVM_MODE_WORKAROUND=0 +export AFL_ENABLE_DICTIONARY=0 + +# Start compiling afl++. +echo "Copying precompiled afl++" + +# Copy afl++ tools necessary for fuzzing. pushd $SRC/aflplusplus > /dev/null -# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it -# down with sanitizers. -INITIAL_CXXFLAGS=$CXXFLAGS -INITIAL_CFLAGS=$CFLAGS -unset CXXFLAGS -unset CFLAGS -make clean -AFL_NO_X86=1 PYTHON_INCLUDE=/ make -CFLAGS=$INITIAL_CFLAGS -CXXFLAGS=$INITIAL_CXXFLAGS - -# Build afl++ driver with existing CFLAGS, CXXFLAGS. -make -C utils/aflpp_driver -cp libAFLDriver.a $LIB_FUZZING_ENGINE +cp -f libAFLDriver.a $LIB_FUZZING_ENGINE # Some important projects include libraries, copy those even when they don't # start with "afl-". Use "sort -u" to avoid a warning about duplicates. ls afl-* *.txt *.a *.o *.so | sort -u | xargs cp -t $OUT -popd > /dev/null - export CC="$SRC/aflplusplus/afl-clang-fast" export CXX="$SRC/aflplusplus/afl-clang-fast++" # Set sane afl++ environment defaults: # Be quiet, otherwise this can break some builds. export AFL_QUIET=1 -# Several targets run their own tools, so ensure its working. -export AFL_MAP_SIZE=4194304 # No leak errors during builds. -export ASAN_OPTIONS="detect_leaks=0:symbolize=0" -# -# Placeholder for the upcoming afl++ build options roulette -# +export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1" + +# AFL compile option roulette. It is OK if they all happen together. + +# 40% chance to perform CMPLOG +rm -f "$OUT/afl_cmplog.txt" +test $(($RANDOM % 10)) -lt 4 && { + export AFL_LLVM_CMPLOG=1 + touch "$OUT/afl_cmplog.txt" +} + +# 10% chance to perform LAF_INTEL +test $(($RANDOM % 10)) -lt 1 && { + export AFL_LLVM_LAF_ALL=1 +} + +# If the targets wants a dictionary - then create one. +test "$AFL_ENABLE_DICTIONARY" = "1" && { + export AFL_LLVM_DICT2FILE="$OUT/afl++.dict" +} + +# In case afl-clang-fast ever breaks, this is a workaround: +test "$AFL_LLVM_MODE_WORKAROUND" = "1" && { + export CC=clang + export CXX=clang++ + WORKAROUND_FLAGS=-fsanitize-coverage=trace-pc-guard + # We can still do CMPLOG light: + test -e "$OUT/afl_cmplog.txt" && { + WORKAROUND_FLAGS="$WORKAROUND_FLAGS",trace-cmp + } + export CFLAGS="$CFLAGS $WORKAROUND_FLAGS" + export CXXFLAGS="$CXXFLAGS $WORKAROUND_FLAGS" + unset AFL_LLVM_LAF_ALL + unset AFL_LLVM_DICT2FILE + unset AFL_ENABLE_DICTIONARY + # We need to create a new fuzzer lib however. + ar ru libAFLDrivernew.a afl-compiler-rt.o utils/aflpp_driver/aflpp_driver.o + cp -f libAFLDrivernew.a $LIB_FUZZING_ENGINE +} + +# Provide a way to document the afl++ options used in this build: +echo +echo afl++ target compilation setup: +env | grep AFL_ | tee "$OUT/afl_options.txt" +echo + +popd > /dev/null echo " done." diff --git a/infra/base-images/base-builder/compile_go_fuzzer b/infra/base-images/base-builder/compile_go_fuzzer index 8f8cde759..2342800fb 100755 --- a/infra/base-images/base-builder/compile_go_fuzzer +++ b/infra/base-images/base-builder/compile_go_fuzzer @@ -18,27 +18,35 @@ path=$1 function=$2 fuzzer=$3 -tags="" +tags="-tags gofuzz" if [[ $# -eq 4 ]]; then tags="-tags $4" fi +# makes directory change temporary +( +cd $GOPATH/src/$path || true +# in the case we are in the right directory, with go.mod but no go.sum +go mod tidy || true +# project was downloaded with go get if go list fails +go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; } +# project does not have go.mod if go list fails again +go list $tags $path || { go mod init $path && go mod tidy ;} + if [[ $SANITIZER = *coverage* ]]; then - cd $GOPATH/src/$path - fuzzed_package=`go list $tags -f '{{.Name}}'` + fuzzed_package=`go list $tags -f '{{.Name}}' $path` + abspath=`go list $tags -f {{.Dir}} $path` + cd $abspath cp $GOPATH/ossfuzz_coverage_runner.go ./"${function,,}"_test.go sed -i -e 's/FuzzFunction/'$function'/' ./"${function,,}"_test.go sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go - echo "#!/bin/sh" > $OUT/$fuzzer - echo "cd $path" >> $OUT/$fuzzer - # The fuzzer may be in a subdirectory, but we want the coverage report for the whole repository fuzzed_repo=`echo $path | cut -d/ -f-3` - echo "go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -coverprofile \$1 " >> $OUT/$fuzzer - chmod +x $OUT/$fuzzer - - cd - + abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` + # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir + echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath + go test -run Test${function}Corpus -v $tags -coverpkg $fuzzed_repo/... -c -o $OUT/$fuzzer $path else # Compile and instrument all Go files relevant to this fuzz target. echo "Running go-fuzz $tags -func $function -o $fuzzer.a $path" @@ -47,3 +55,4 @@ else # Link Go code ($fuzzer.a) with fuzzing engine to produce fuzz target binary. $CXX $CXXFLAGS $LIB_FUZZING_ENGINE $fuzzer.a -o $OUT/$fuzzer fi +) diff --git a/infra/base-images/base-builder/compile_honggfuzz b/infra/base-images/base-builder/compile_honggfuzz index 362a0a598..f86e8426d 100755 --- a/infra/base-images/base-builder/compile_honggfuzz +++ b/infra/base-images/base-builder/compile_honggfuzz @@ -17,8 +17,8 @@ echo "Skipping compilation; using precompiled honggfuzz" -cp $PRECOMPILED_DIR/honggfuzz.a $LIB_FUZZING_ENGINE -cp $PRECOMPILED_DIR/honggfuzz $OUT/ +cp $SRC/honggfuzz/honggfuzz.a $LIB_FUZZING_ENGINE +cp $SRC/honggfuzz/honggfuzz $OUT/ # Custom coverage flags, roughly in sync with: # https://github.com/google/honggfuzz/blob/oss-fuzz/hfuzz_cc/hfuzz-cc.c diff --git a/infra/base-images/base-builder/compile_libfuzzer b/infra/base-images/base-builder/compile_libfuzzer index 00f2d6337..3fd7f3906 100755 --- a/infra/base-images/base-builder/compile_libfuzzer +++ b/infra/base-images/base-builder/compile_libfuzzer @@ -16,16 +16,7 @@ ################################################################################ echo -n "Compiling libFuzzer to $LIB_FUZZING_ENGINE... " -mkdir -p $WORK/libfuzzer -pushd $WORK/libfuzzer > /dev/null - -# Use -fPIC to allow preloading (LD_PRELOAD). -$CXX $CXXFLAGS -std=c++11 -O2 -fPIC $SANITIZER_FLAGS -fno-sanitize=vptr \ - -c $SRC/libfuzzer/*.cpp -I$SRC/libfuzzer -ar r $LIB_FUZZING_ENGINE_DEPRECATED $WORK/libfuzzer/*.o -popd > /dev/null -rm -rf $WORK/libfuzzer -# Override variable as libFuzzer builds do not link directly against an -# engine library, but use -fsanitize=fuzzer to instruct clang to do so. export LIB_FUZZING_ENGINE="-fsanitize=fuzzer" +cp /usr/local/lib/clang/*/lib/linux/libclang_rt.fuzzer-$ARCHITECTURE.a \ + $LIB_FUZZING_ENGINE_DEPRECATED echo " done." diff --git a/infra/base-images/base-builder/debug_afl b/infra/base-images/base-builder/debug_afl new file mode 100755 index 000000000..c53dae815 --- /dev/null +++ b/infra/base-images/base-builder/debug_afl @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Source this file for afl++ debug sessions. +apt-get update +apt-get install -y strace gdb vim joe psmisc + +pushd $SRC/aflplusplus > /dev/null +git checkout dev +git pull +test -n "$1" && { git checkout "$1" ; git pull ; } +CFLAGS_SAVE="$CFLAGS" +CXXFLAGS_SAVE="$CXXFLAGS" +unset CFLAGS +unset CXXFLAGS +make +export CFLAGS="$CFLAGS_SAVE" +export CXXFLAGS="$CXXFLAGS_SAVE" +popd > /dev/null + +export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_error=1" +export AFL_LLVM_LAF_ALL=1 +export AFL_LLVM_CMPLOG=1 +touch "$OUT/afl_cmplog.txt" +export AFL_LLVM_DICT2FILE=$OUT/afl++.dict +ulimit -c unlimited diff --git a/infra/base-images/base-builder/detect_repo.py b/infra/base-images/base-builder/detect_repo.py index 8969e974f..e677e1023 100644 --- a/infra/base-images/base-builder/detect_repo.py +++ b/infra/base-images/base-builder/detect_repo.py @@ -107,20 +107,25 @@ def get_repo(repo_path): return None -def check_for_repo_name(repo_path, repo_name): - """Check to see if the repo_name matches the remote repository repo name. +def check_for_repo_name(repo_path, expected_repo_name): + """Returns True if the repo at |repo_path| repo_name matches + |expected_repo_name|. Args: - repo_path: The directory of the git repo. - repo_name: The name of the target git repo. + repo_path: The directory of a git repo. + expected_repo_name: The name of the target git repo. """ if not os.path.exists(os.path.join(repo_path, '.git')): return False - out, _ = execute(['git', 'config', '--get', 'remote.origin.url'], - location=repo_path) - out = out.split('/')[-1].replace('.git', '').rstrip() - return out == repo_name + repo_url, _ = execute(['git', 'config', '--get', 'remote.origin.url'], + location=repo_path) + # Handle two common cases: + # https://github.com/google/syzkaller/ + # https://github.com/google/syzkaller.git + repo_url = repo_url.replace('.git', '').rstrip().rstrip('/') + actual_repo_name = repo_url.split('/')[-1] + return actual_repo_name == expected_repo_name def check_for_commit(repo_path, commit): diff --git a/infra/base-images/base-builder/detect_repo_test.py b/infra/base-images/base-builder/detect_repo_test.py index 21f64af44..0243b3ac5 100644 --- a/infra/base-images/base-builder/detect_repo_test.py +++ b/infra/base-images/base-builder/detect_repo_test.py @@ -23,6 +23,7 @@ import re import sys import tempfile import unittest +from unittest import mock import detect_repo @@ -36,6 +37,33 @@ import test_repos # pylint: enable=wrong-import-position +class TestCheckForRepoName(unittest.TestCase): + """Tests for check_for_repo_name.""" + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('detect_repo.execute', + return_value=('https://github.com/google/syzkaller/', None)) + def test_go_get_style_url(self, _, __): + """Tests that check_for_repo_name works on repos that were downloaded using + go get.""" + self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller')) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('detect_repo.execute', + return_value=('https://github.com/google/syzkaller', None)) + def test_missing_git_and_slash_url(self, _, __): + """Tests that check_for_repo_name works on repos who's URLs do not end in + ".git" or "/".""" + self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller')) + + @mock.patch('os.path.exists', return_value=True) + @mock.patch('detect_repo.execute', + return_value=('https://github.com/google/syzkaller.git', None)) + def test_normal_style_repo_url(self, _, __): + """Tests that check_for_repo_name works on normally cloned repos.""" + self.assertTrue(detect_repo.check_for_repo_name('fake-path', 'syzkaller')) + + @unittest.skipIf(not os.getenv('INTEGRATION_TESTS'), 'INTEGRATION_TESTS=1 not set') class DetectRepoIntegrationTest(unittest.TestCase): diff --git a/infra/base-images/base-builder/precompile_afl b/infra/base-images/base-builder/precompile_afl new file mode 100755 index 000000000..d6e71f2c3 --- /dev/null +++ b/infra/base-images/base-builder/precompile_afl @@ -0,0 +1,35 @@ +#!/bin/bash -eu +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +echo "Precompiling AFLplusplus" + +pushd $SRC/aflplusplus > /dev/null +make clean +# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it +# down with sanitizers. +SAVE_CXXFLAGS=$CXXFLAGS +SAVE_CFLAGS=$CFLAGS +unset CXXFLAGS +unset CFLAGS +export AFL_IGNORE_UNKNOWN_ENVS=1 +make clean +AFL_NO_X86=1 PYTHON_INCLUDE=/ make +make -C utils/aflpp_driver + +popd > /dev/null + +echo "Done." diff --git a/infra/base-images/base-builder/precompile_honggfuzz b/infra/base-images/base-builder/precompile_honggfuzz index 2565bb83f..df6bb2b75 100755 --- a/infra/base-images/base-builder/precompile_honggfuzz +++ b/infra/base-images/base-builder/precompile_honggfuzz @@ -15,7 +15,7 @@ # ################################################################################ -echo -n "Precompiling honggfuzz to $PRECOMPILED_DIR..." +echo "Precompiling honggfuzz" export BUILD_OSSFUZZ_STATIC=true PACKAGES=( @@ -37,10 +37,9 @@ CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD" make # libhfuzz.a will be added by CC/CXX linker directly during linking, # but it's defined here to satisfy the build infrastructure -ar rcs $PRECOMPILED_DIR/honggfuzz.a libhfuzz/*.o libhfcommon/*.o -cp honggfuzz $PRECOMPILED_DIR/ +ar rcs honggfuzz.a libhfuzz/*.o libhfcommon/*.o popd > /dev/null apt-get remove -y --purge ${PACKAGES[@]} apt-get autoremove -y -echo " done." +echo "Done." diff --git a/infra/base-images/base-clang/Dockerfile b/infra/base-images/base-clang/Dockerfile index 928e7934f..3c16a8f3c 100644 --- a/infra/base-images/base-clang/Dockerfile +++ b/infra/base-images/base-clang/Dockerfile @@ -25,7 +25,8 @@ RUN apt-get update && apt-get install -y wget sudo && \ chmod +x cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ ./cmake-$CMAKE_VERSION-Linux-x86_64.sh --skip-license --prefix="/usr/local" && \ rm cmake-$CMAKE_VERSION-Linux-x86_64.sh && \ - SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo + SUDO_FORCE_REMOVE=yes apt-get remove --purge -y wget sudo && \ + rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui COPY checkout_build_install_llvm.sh /root/ # Keep all steps in the same script to decrease the number of intermediate diff --git a/infra/base-images/base-clang/checkout_build_install_llvm.sh b/infra/base-images/base-clang/checkout_build_install_llvm.sh index c5d97d5bf..f6e8ca99c 100755 --- a/infra/base-images/base-clang/checkout_build_install_llvm.sh +++ b/infra/base-images/base-clang/checkout_build_install_llvm.sh @@ -15,10 +15,14 @@ # ################################################################################ -NPROC=16 # See issue #4270. The compiler crashes on GCB instance with 32 vCPUs. +# See issue #4270. The compiler crashes on GCB instance with 32 vCPUs, so when +# we compile on GCB we want 16 cores. But locally we want more (so use nproc / +# 2). +NPROC=$(expr $(nproc) / 2) -LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev" -apt-get install -y $LLVM_DEP_PACKAGES +# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler +LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev" +apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends # Checkout CHECKOUT_RETRIES=10 @@ -60,7 +64,7 @@ function cmake_llvm { # Use chromium's clang revision mkdir $SRC/chromium_tools cd $SRC/chromium_tools -git clone https://chromium.googlesource.com/chromium/src/tools/clang +git clone https://chromium.googlesource.com/chromium/src/tools/clang --depth 1 cd clang LLVM_SRC=$SRC/llvm-project @@ -89,11 +93,9 @@ fi git -C $LLVM_SRC checkout $LLVM_REVISION echo "Using LLVM revision: $LLVM_REVISION" -# Build & install. We build clang in two stages because gcc can't build a -# static version of libcxxabi -# (see https://github.com/google/oss-fuzz/issues/2164). +# Build & install. mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1 -cd $WORK/llvm-stage1 +python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1 TARGET_TO_BUILD= case $(uname -m) in @@ -111,9 +113,6 @@ esac PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld" -cmake_llvm -ninja -j $NPROC - cd $WORK/llvm-stage2 export CC=$WORK/llvm-stage1/bin/clang export CXX=$WORK/llvm-stage1/bin/clang++ @@ -175,3 +174,54 @@ rm -rf $LLVM_SRC rm -rf $SRC/chromium_tools apt-get remove --purge -y $LLVM_DEP_PACKAGES apt-get autoremove -y + +# Delete unneeded parts of LLVM to reduce image size. +# See https://github.com/google/oss-fuzz/issues/5170 +LLVM_TOOLS_TMPDIR=/tmp/llvm-tools +mkdir $LLVM_TOOLS_TMPDIR +# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR +mv \ + /usr/local/bin/llvm-ar \ + /usr/local/bin/llvm-as \ + /usr/local/bin/llvm-config \ + /usr/local/bin/llvm-cov \ + /usr/local/bin/llvm-objcopy \ + /usr/local/bin/llvm-profdata \ + /usr/local/bin/llvm-ranlib \ + /usr/local/bin/llvm-symbolizer \ + /usr/local/bin/llvm-undname \ + $LLVM_TOOLS_TMPDIR +# Delete remaining llvm- binaries. +rm -rf /usr/local/bin/llvm-* +# Restore the llvm- binaries we want to keep. +mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/ +rm -rf $LLVM_TOOLS_TMPDIR + +# Remove binaries from LLVM buld that we don't need. +rm -f \ + /usr/local/bin/bugpoint \ + /usr/local/bin/llc \ + /usr/local/bin/lli \ + /usr/local/bin/clang-check \ + /usr/local/bin/clang-refactor \ + /usr/local/bin/clang-offload-wrapper \ + /usr/local/bin/clang-offload-bundler \ + /usr/local/bin/clang-check \ + /usr/local/bin/clang-refactor \ + /usr/local/bin/c-index-test \ + /usr/local/bin/clang-rename \ + /usr/local/bin/clang-scan-deps \ + /usr/local/bin/clang-extdef-mapping \ + /usr/local/bin/diagtool \ + /usr/local/bin/sanstats \ + /usr/local/bin/dsymutil \ + /usr/local/bin/verify-uselistorder \ + /usr/local/bin/clang-format + +# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the +# libraries. +# Note: we need fuzzer_no_main libraries for atheris. Don't delete. +rm -rf \ + /usr/local/lib/libclang* \ + /usr/local/lib/liblld* \ + /usr/local/lib/cmake/ diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile index f0a264fa7..f847de026 100644..100755 --- a/infra/base-images/base-runner/Dockerfile +++ b/infra/base-images/base-runner/Dockerfile @@ -14,30 +14,40 @@ # ################################################################################ +# Build rust stuff in its own image. We only need the resulting binaries. +# Keeping the rust toolchain in the image wastes 1 GB. +FROM gcr.io/oss-fuzz-base/base-image as temp-runner-binary-builder + +RUN apt-get update && apt-get install -y cargo +RUN cargo install rustfilt + # Using multi-stage build to copy some LLVM binaries needed in the runner image. FROM gcr.io/oss-fuzz-base/base-clang AS base-clang +# Real image that will be used later. FROM gcr.io/oss-fuzz-base/base-image +COPY --from=temp-runner-binary-builder /root/.cargo/bin/rustfilt /usr/local/bin + # Copy the binaries needed for code coverage and crash symbolization. COPY --from=base-clang /usr/local/bin/llvm-cov \ /usr/local/bin/llvm-profdata \ /usr/local/bin/llvm-symbolizer \ /usr/local/bin/ -# TODO(metzman): Install libc6-i386 lib32gcc1 instead of libc6-dev-i386 for -# consistency with ClusterFuzz image and to reduce size. RUN apt-get update && apt-get install -y \ binutils \ file \ fonts-dejavu \ git \ - libc6-dev-i386 \ + lib32gcc1 \ + libc6-i386 \ libcap2 \ python3 \ python3-pip \ + unzip \ wget \ - zip + zip --no-install-recommends RUN git clone https://chromium.googlesource.com/chromium/src/tools/code_coverage /opt/code_coverage && \ pip3 install -r /opt/code_coverage/requirements.txt @@ -66,7 +76,20 @@ ENV GOPATH /root/go ENV PATH $PATH:/root/.go/bin:$GOPATH/bin # Set up Golang coverage modules. -RUN go get github.com/google/oss-fuzz/infra/go/coverage/... +COPY gocoverage $GOPATH/gocoverage +RUN cd $GOPATH/gocoverage && go install ./... + +# Install OpenJDK 15 and trim its size by removing unused components. +ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64 +ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server +ENV PATH=$PATH:$JAVA_HOME/bin + +RUN wget https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz -O /tmp/openjdk-15.0.2_linux-x64_bin.tar.gz && \ + cd /tmp && \ + mkdir -p $JAVA_HOME && \ + tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ + rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \ + rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip # Do this last to make developing these files easier/faster due to caching. COPY bad_build_check \ @@ -75,12 +98,11 @@ COPY bad_build_check \ coverage_helper \ dataflow_tracer.py \ download_corpus \ - minijail0 \ + rcfilt \ reproduce \ run_fuzzer \ - run_minijail \ parse_options.py \ targets_list \ test_all.py \ - test_one \ + test_one.py \ /usr/local/bin/ diff --git a/infra/base-images/base-runner/bad_build_check b/infra/base-images/base-runner/bad_build_check index a57a48252..01f8fbbab 100755 --- a/infra/base-images/base-runner/bad_build_check +++ b/infra/base-images/base-runner/bad_build_check @@ -301,6 +301,12 @@ function check_mixed_sanitizers { local result=0 local CALL_INSN= + if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then + # Sanitizer runtime is linked into the Jazzer driver, so this check does not + # apply. + return 0 + fi + if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then # Sanitizer runtime is loaded via LD_PRELOAD, so this check does not apply. return 0 @@ -312,7 +318,7 @@ function check_mixed_sanitizers { else case $(uname -m) in x86_64) - CALL_INSN="callq\s+[0-9a-f]+\s+<" + CALL_INSN="callq?\s+[0-9a-f]+\s+<" ;; aarch64) CALL_INSN="bl\s+[0-9a-f]+\s+<" @@ -328,6 +334,7 @@ function check_mixed_sanitizers { local MSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__msan" -c) local UBSAN_CALLS=$(objdump -dC $FUZZER | egrep "${CALL_INSN}__ubsan" -c) + if [[ "$SANITIZER" = address ]]; then check_asan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS result=$? @@ -340,6 +347,9 @@ function check_mixed_sanitizers { elif [[ "$SANITIZER" = undefined ]]; then check_ubsan_build $FUZZER $ASAN_CALLS $DFSAN_CALLS $MSAN_CALLS $UBSAN_CALLS result=$? + elif [[ "$SANITIZER" = thread ]]; then + # TODO(metzman): Implement this. + result=0 fi return $result @@ -376,6 +386,12 @@ function check_architecture { local FUZZER=$1 local FUZZER_NAME=$(basename $FUZZER) + if [ "${FUZZING_LANGUAGE:-}" = "jvm" ]; then + # The native dependencies of a JVM project are not packaged, but loaded + # dynamically at runtime and thus cannot be checked here. + return 0; + fi + if [ "${FUZZING_LANGUAGE:-}" = "python" ]; then FUZZER=${FUZZER}.pkg fi diff --git a/infra/base-images/base-runner/coverage b/infra/base-images/base-runner/coverage index 2fcf9e977..a86b00dec 100755 --- a/infra/base-images/base-runner/coverage +++ b/infra/base-images/base-runner/coverage @@ -114,13 +114,15 @@ function run_fuzz_target { function run_go_fuzz_target { local target=$1 - cd $GOPATH/src echo "Running go target $target" export FUZZ_CORPUS_DIR="/corpus/${target}/" export FUZZ_PROFILE_NAME="$DUMPS_DIR/$target.perf" - bash $OUT/$target $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log + $OUT/$target -test.coverprofile $DUMPS_DIR/$target.profdata &> $LOGS_DIR/$target.log + # translate from golangish paths to current absolute paths + cat $OUT/$target.gocovpath | while read i; do sed -i $i $DUMPS_DIR/$target.profdata; done + # cf PATH_EQUIVALENCE_ARGS + sed -i 's=/='$OUT'/=' $DUMPS_DIR/$target.profdata $SYSGOPATH/bin/gocovsum $DUMPS_DIR/$target.profdata > $FUZZER_STATS_DIR/$target.json - cd $OUT } export SYSGOPATH=$GOPATH @@ -131,7 +133,7 @@ for fuzz_target in $FUZZ_TARGETS; do if [[ $FUZZING_LANGUAGE == "go" ]]; then # Continue if not a fuzz target. if [[ $FUZZING_ENGINE != "none" ]]; then - grep "go test -run" $fuzz_target > /dev/null 2>&1 || continue + grep "FUZZ_CORPUS_DIR" $fuzz_target > /dev/null 2>&1 || continue fi run_go_fuzz_target $fuzz_target & else @@ -193,7 +195,7 @@ else # Generate HTML report. llvm-cov show -format=html -output-dir=$REPORT_ROOT_DIR \ - -Xdemangler c++filt -Xdemangler -n $LLVM_COV_ARGS + -Xdemangler rcfilt $LLVM_COV_ARGS # Export coverage summary in JSON format. llvm-cov export -summary-only $LLVM_COV_ARGS > $SUMMARY_FILE diff --git a/infra/base-images/base-runner/gocoverage/go.mod b/infra/base-images/base-runner/gocoverage/go.mod new file mode 100644 index 000000000..b0b57216e --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/go.mod @@ -0,0 +1,8 @@ +module oss-fuzz.com/gocoverage + +go 1.14 + +require ( + github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 + golang.org/x/tools v0.1.0 +) diff --git a/infra/base-images/base-runner/gocoverage/go.sum b/infra/base-images/base-runner/gocoverage/go.sum new file mode 100644 index 000000000..3279af3ba --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/go.sum @@ -0,0 +1,30 @@ +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5 h1:zIaiqGYDQwa4HVx5wGRTXbx38Pqxjemn4BP98wpzpXo= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE b/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE new file mode 100644 index 000000000..455fb1087 --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/gocovmerge/LICENSE @@ -0,0 +1,22 @@ +Copyright (c) 2015, Wade Simmons +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go b/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go new file mode 100644 index 000000000..e8099839e --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/gocovmerge/gocovmerge.go @@ -0,0 +1,111 @@ +// gocovmerge takes the results from multiple `go test -coverprofile` runs and +// merges them into one profile +package main + +import ( + "flag" + "fmt" + "io" + "log" + "os" + "sort" + + "golang.org/x/tools/cover" +) + +func mergeProfiles(p *cover.Profile, merge *cover.Profile) { + if p.Mode != merge.Mode { + log.Fatalf("cannot merge profiles with different modes") + } + // Since the blocks are sorted, we can keep track of where the last block + // was inserted and only look at the blocks after that as targets for merge + startIndex := 0 + for _, b := range merge.Blocks { + startIndex = mergeProfileBlock(p, b, startIndex) + } +} + +func mergeProfileBlock(p *cover.Profile, pb cover.ProfileBlock, startIndex int) int { + sortFunc := func(i int) bool { + pi := p.Blocks[i+startIndex] + return pi.StartLine >= pb.StartLine && (pi.StartLine != pb.StartLine || pi.StartCol >= pb.StartCol) + } + + i := 0 + if sortFunc(i) != true { + i = sort.Search(len(p.Blocks)-startIndex, sortFunc) + } + i += startIndex + if i < len(p.Blocks) && p.Blocks[i].StartLine == pb.StartLine && p.Blocks[i].StartCol == pb.StartCol { + if p.Blocks[i].EndLine != pb.EndLine || p.Blocks[i].EndCol != pb.EndCol { + log.Fatalf("OVERLAP MERGE: %v %v %v", p.FileName, p.Blocks[i], pb) + } + switch p.Mode { + case "set": + p.Blocks[i].Count |= pb.Count + case "count", "atomic": + p.Blocks[i].Count += pb.Count + default: + log.Fatalf("unsupported covermode: '%s'", p.Mode) + } + } else { + if i > 0 { + pa := p.Blocks[i-1] + if pa.EndLine >= pb.EndLine && (pa.EndLine != pb.EndLine || pa.EndCol > pb.EndCol) { + log.Fatalf("OVERLAP BEFORE: %v %v %v", p.FileName, pa, pb) + } + } + if i < len(p.Blocks)-1 { + pa := p.Blocks[i+1] + if pa.StartLine <= pb.StartLine && (pa.StartLine != pb.StartLine || pa.StartCol < pb.StartCol) { + log.Fatalf("OVERLAP AFTER: %v %v %v", p.FileName, pa, pb) + } + } + p.Blocks = append(p.Blocks, cover.ProfileBlock{}) + copy(p.Blocks[i+1:], p.Blocks[i:]) + p.Blocks[i] = pb + } + return i + 1 +} + +func addProfile(profiles []*cover.Profile, p *cover.Profile) []*cover.Profile { + i := sort.Search(len(profiles), func(i int) bool { return profiles[i].FileName >= p.FileName }) + if i < len(profiles) && profiles[i].FileName == p.FileName { + mergeProfiles(profiles[i], p) + } else { + profiles = append(profiles, nil) + copy(profiles[i+1:], profiles[i:]) + profiles[i] = p + } + return profiles +} + +func dumpProfiles(profiles []*cover.Profile, out io.Writer) { + if len(profiles) == 0 { + return + } + fmt.Fprintf(out, "mode: %s\n", profiles[0].Mode) + for _, p := range profiles { + for _, b := range p.Blocks { + fmt.Fprintf(out, "%s:%d.%d,%d.%d %d %d\n", p.FileName, b.StartLine, b.StartCol, b.EndLine, b.EndCol, b.NumStmt, b.Count) + } + } +} + +func main() { + flag.Parse() + + var merged []*cover.Profile + + for _, file := range flag.Args() { + profiles, err := cover.ParseProfiles(file) + if err != nil { + log.Fatalf("failed to parse profiles: %v", err) + } + for _, p := range profiles { + merged = addProfile(merged, p) + } + } + + dumpProfiles(merged, os.Stdout) +} diff --git a/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go b/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go new file mode 100644 index 000000000..973b7ae92 --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/gocovsum/gocovsum.go @@ -0,0 +1,147 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "log" + + "go/ast" + "go/parser" + "go/token" + + "golang.org/x/tools/cover" +) + +type CoverageTotal struct { + Count int `json:"count"` + Covered int `json:"covered"` + Uncovered int `json:"notcovered"` + Percent float64 `json:"percent"` +} + +type CoverageTotals struct { + Functions CoverageTotal `json:"functions,omitempty"` + Lines CoverageTotal `json:"lines,omitempty"` + Regions CoverageTotal `json:"regions,omitempty"` + Instantiations CoverageTotal `json:"instantiations,omitempty"` + Branches CoverageTotal `json:"branches,omitempty"` +} + +type CoverageFile struct { + Summary CoverageTotals `json:"summary,omitempty"` + Filename string `json:"filename,omitempty"` +} + +type CoverageData struct { + Totals CoverageTotals `json:"totals,omitempty"` + Files []CoverageFile `json:"files,omitempty"` +} + +type PositionInterval struct { + start token.Position + end token.Position +} + +type CoverageSummary struct { + Data []CoverageData `json:"data,omitempty"` + Type string `json:"type,omitempty"` + Version string `json:"version,omitempty"` +} + +func isFunctionCovered(s token.Position, e token.Position, blocks []cover.ProfileBlock) bool { + for _, b := range blocks { + if b.StartLine >= s.Line && b.StartLine <= e.Line && b.EndLine >= s.Line && b.EndLine <= e.Line { + if b.Count > 0 { + return true + } + } + } + return false +} + +func computePercent(s *CoverageTotals) { + s.Regions.Percent = float64(100*s.Regions.Covered) / float64(s.Regions.Count) + s.Lines.Percent = float64(100*s.Lines.Covered) / float64(s.Lines.Count) + s.Functions.Percent = float64(100*s.Functions.Covered) / float64(s.Functions.Count) +} + +func main() { + flag.Parse() + + if len(flag.Args()) != 1 { + log.Fatalf("needs exactly one argument") + } + profiles, err := cover.ParseProfiles(flag.Args()[0]) + if err != nil { + log.Fatalf("failed to parse profiles: %v", err) + } + r := CoverageSummary{} + r.Type = "oss-fuzz.go.coverage.json.export" + r.Version = "2.0.1" + r.Data = make([]CoverageData, 1) + for _, p := range profiles { + fset := token.NewFileSet() // positions are relative to fset + f, err := parser.ParseFile(fset, p.FileName, nil, 0) + if err != nil { + panic(err) + } + fileCov := CoverageFile{} + fileCov.Filename = p.FileName + ast.Inspect(f, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.FuncLit: + startf := fset.Position(x.Pos()) + endf := fset.Position(x.End()) + fileCov.Summary.Functions.Count++ + if isFunctionCovered(startf, endf, p.Blocks) { + fileCov.Summary.Functions.Covered++ + } else { + fileCov.Summary.Functions.Uncovered++ + } + case *ast.FuncDecl: + startf := fset.Position(x.Pos()) + endf := fset.Position(x.End()) + fileCov.Summary.Functions.Count++ + if isFunctionCovered(startf, endf, p.Blocks) { + fileCov.Summary.Functions.Covered++ + } else { + fileCov.Summary.Functions.Uncovered++ + } + } + return true + }) + + for _, b := range p.Blocks { + fileCov.Summary.Regions.Count++ + if b.Count > 0 { + fileCov.Summary.Regions.Covered++ + } else { + fileCov.Summary.Regions.Uncovered++ + } + + fileCov.Summary.Lines.Count += b.NumStmt + if b.Count > 0 { + fileCov.Summary.Lines.Covered += b.NumStmt + } else { + fileCov.Summary.Lines.Uncovered += b.NumStmt + } + } + r.Data[0].Totals.Regions.Count += fileCov.Summary.Regions.Count + r.Data[0].Totals.Regions.Covered += fileCov.Summary.Regions.Covered + r.Data[0].Totals.Regions.Uncovered += fileCov.Summary.Regions.Uncovered + r.Data[0].Totals.Lines.Count += fileCov.Summary.Lines.Count + r.Data[0].Totals.Lines.Covered += fileCov.Summary.Lines.Covered + r.Data[0].Totals.Lines.Uncovered += fileCov.Summary.Lines.Uncovered + r.Data[0].Totals.Functions.Count += fileCov.Summary.Functions.Count + r.Data[0].Totals.Functions.Covered += fileCov.Summary.Functions.Covered + r.Data[0].Totals.Functions.Uncovered += fileCov.Summary.Functions.Uncovered + + computePercent(&fileCov.Summary) + r.Data[0].Files = append(r.Data[0].Files, fileCov) + } + + computePercent(&r.Data[0].Totals) + o, _ := json.Marshal(r) + fmt.Printf(string(o)) +} diff --git a/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE b/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/pprof-merge/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/infra/base-images/base-runner/gocoverage/pprof-merge/main.go b/infra/base-images/base-runner/gocoverage/pprof-merge/main.go new file mode 100644 index 000000000..f35156403 --- /dev/null +++ b/infra/base-images/base-runner/gocoverage/pprof-merge/main.go @@ -0,0 +1,68 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "log" + "os" + + "github.com/google/pprof/profile" +) + +var ( + output string +) + +func main() { + flag.StringVar(&output, "o", "merged.data", "") + flag.Parse() + + files := os.Args[1:] + if len(files) == 0 { + log.Fatal("Give profiles files as arguments") + } + + var profiles []*profile.Profile + for _, fname := range files { + f, err := os.Open(fname) + if err != nil { + log.Fatalf("Cannot open profile file at %q: %v", fname, err) + } + p, err := profile.Parse(f) + if err != nil { + log.Fatalf("Cannot parse profile at %q: %v", fname, err) + } + profiles = append(profiles, p) + } + + merged, err := profile.Merge(profiles) + if err != nil { + log.Fatalf("Cannot merge profiles: %v", err) + } + + out, err := os.OpenFile(output, os.O_RDWR|os.O_CREATE, 0755) + if err != nil { + log.Fatalf("Cannot open output to write: %v", err) + } + + if err := merged.Write(out); err != nil { + log.Fatalf("Cannot write merged profile to file: %v", err) + } + + if err := out.Close(); err != nil { + log.Printf("Error when closing the output file: %v", err) + } +} diff --git a/infra/base-images/base-runner/minijail0 b/infra/base-images/base-runner/minijail0 Binary files differdeleted file mode 100755 index 369e0bbd9..000000000 --- a/infra/base-images/base-runner/minijail0 +++ /dev/null diff --git a/infra/base-images/base-runner/rcfilt b/infra/base-images/base-runner/rcfilt new file mode 100755 index 000000000..1c621100c --- /dev/null +++ b/infra/base-images/base-runner/rcfilt @@ -0,0 +1,21 @@ +#!/bin/bash -u +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Symbol demangling for both C++ and Rust +# +################################################################################ + +# simply pipe +rustfilt | c++filt -n diff --git a/infra/base-images/base-runner/run_fuzzer b/infra/base-images/base-runner/run_fuzzer index 6464ddc2c..b9bc8d9d6 100755 --- a/infra/base-images/base-runner/run_fuzzer +++ b/infra/base-images/base-runner/run_fuzzer @@ -98,22 +98,29 @@ fi if [[ "$FUZZING_ENGINE" = afl ]]; then # Set afl++ environment options. - export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1:symbolize=0" + export ASAN_OPTIONS="$ASAN_OPTIONS:abort_on_error=1:symbolize=0:detect_odr_violation=0:" export MSAN_OPTIONS="$MSAN_OPTIONS:exit_code=86:symbolize=0" export UBSAN_OPTIONS="$UBSAN_OPTIONS:symbolize=0" export AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 export AFL_SKIP_CPUFREQ=1 export AFL_NO_AFFINITY=1 export AFL_FAST_CAL=1 - export AFL_MAP_SIZE=4194304 # If $OUT/afl_cmplog.txt is present this means the target was compiled for # CMPLOG. So we have to add the proper parameters to afl-fuzz. `-l 2` is # CMPLOG level 2, which will colorize larger files but not huge files and # not enable transform analysis unless there have been several cycles without # any finds. - test -e $OUT/afl_cmplog.txt && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -l 2 -c $OUT/$FUZZER" + test -e "$OUT/afl_cmplog.txt" && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -l 2 -c $OUT/$FUZZER" + # If $OUT/afl++.dict we load it as a dictionary for afl-fuzz. + test -e "$OUT/afl++.dict" && AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -x $OUT/afl++.dict" + # Ensure timeout is a bit large than 1sec as some of the OSS-Fuzz fuzzers + # are slower than this. + AFL_FUZZER_ARGS="$AFL_FUZZER_ARGS -t 5000+" # AFL expects at least 1 file in the input dir. echo input > ${CORPUS_DIR}/input + echo afl++ setup: + env|grep AFL_ + cat "$OUT/afl_options.txt" CMD_LINE="$OUT/afl-fuzz $AFL_FUZZER_ARGS -i $CORPUS_DIR -o $FUZZER_OUT $(get_dictionary) $* -- $OUT/$FUZZER" elif [[ "$FUZZING_ENGINE" = honggfuzz ]]; then diff --git a/infra/base-images/base-runner/run_minijail b/infra/base-images/base-runner/run_minijail deleted file mode 100755 index bf950b1e7..000000000 --- a/infra/base-images/base-runner/run_minijail +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash -eu -# Copyright 2017 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -rm -rf /tmp/chroot -mkdir /tmp/chroot - -mkdir /tmp/chroot/lib -mkdir /tmp/chroot/lib64 -mkdir /tmp/chroot/lib32 - -mkdir /tmp/chroot/usr -mkdir /tmp/chroot/usr/lib -mkdir /tmp/chroot/usr/lib32 - -mkdir /tmp/chroot/dev -mknod -m 666 /tmp/chroot/dev/null c 1 3 -mknod -m 666 /tmp/chroot/dev/urandom c 1 9 - -mkdir /tmp/chroot/proc -mkdir /tmp/chroot/tmp - -mkdir /tmp/chroot/bin -cp /bin/sh /tmp/chroot/bin/sh -cp $(which llvm-symbolizer) /tmp/chroot/bin/llvm-symbolizer - -FULL_EXE_PATH=$(readlink -f $1) -EXE_DIR=$(dirname $FULL_EXE_PATH) -mkdir -p /tmp/chroot/$EXE_DIR - -shift - -echo 'Running:' -echo minijail0 -U -m \"0 $UID 1\" -T static \ - -c 0 -n -v -p -l -I \ - -k proc,/proc,proc,1 -P /tmp/chroot \ - -b /lib,/lib,0 -b /lib64,/lib64,0 -b /lib32,/lib32,0 -b /usr/lib,/usr/lib,0 \ - -b /usr/lib32,/usr/lib32,0 -b /tmp,/tmp,1 \ - -b $EXE_DIR,$EXE_DIR,0 $FULL_EXE_PATH $@ - -minijail0 -U -m "0 $UID 1" -T static \ - -c 0 -n -v -p -l -I \ - -k proc,/proc,proc,1 -P /tmp/chroot \ - -b /lib,/lib,0 -b /lib64,/lib64,0 -b /lib32,/lib32,0 -b /usr/lib,/usr/lib,0 \ - -b /usr/lib32,/usr/lib32,0 -b /tmp,/tmp,1 \ - -b $EXE_DIR,$EXE_DIR,0 $FULL_EXE_PATH $@ diff --git a/infra/base-images/base-runner/test_all.py b/infra/base-images/base-runner/test_all.py index 360da0345..925ebde69 100755 --- a/infra/base-images/base-runner/test_all.py +++ b/infra/base-images/base-runner/test_all.py @@ -78,11 +78,16 @@ def find_fuzz_targets(directory, fuzzing_language): continue if filename.startswith('afl-'): continue + if filename.startswith('jazzer_'): + continue if not os.path.isfile(path): continue if not os.stat(path).st_mode & EXECUTABLE: continue - if fuzzing_language != 'python' and not is_elf(path): + # Fuzz targets are expected to be ELF binaries for languages other than + # Python and Java. + if (fuzzing_language != 'python' and fuzzing_language != 'jvm' and + not is_elf(path)): continue if os.getenv('FUZZING_ENGINE') != 'none': with open(path, 'rb') as file_handle: diff --git a/infra/base-images/base-runner/test_one b/infra/base-images/base-runner/test_one deleted file mode 100755 index 23b7fd932..000000000 --- a/infra/base-images/base-runner/test_one +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -u -# Copyright 2020 Google Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -################################################################################ - -# Wrapper around bad_build_check that moves the /out directory to /tmp/not-out. -# This is useful when bad_build_check isn't called from test_all which does the -# same thing. - -function main { - # Move the directory the fuzzer is located in to somewhere that doesn't exist - # on the builder to make it more likely that hardcoding /out fails here (since - # it will fail on ClusterFuzz). - local fuzzer=$1 - fuzzer=$(realpath $fuzzer) - local initial_fuzzer_dir=$(dirname $fuzzer) - - local tmp_fuzzer_dir=/tmp/not-out - rm -rf $tmp_fuzzer_dir - mkdir $tmp_fuzzer_dir - # Move the contents of $initial_fuzzer_dir rather than the directory itself in - # case it is a mount. - mv $initial_fuzzer_dir/* $tmp_fuzzer_dir - fuzzer="$tmp_fuzzer_dir/$(basename $fuzzer)" - - # Change OUT to the temporary fuzzer dir. - local initial_out=$OUT - export OUT=$tmp_fuzzer_dir - - bad_build_check $fuzzer - returncode=$? - - # Restore OUT and $initial_fuzzer_dir - export OUT=$initial_out - mv $tmp_fuzzer_dir/* $initial_fuzzer_dir - - return $returncode -} - -if [ $# -ne 1 ]; then - echo "Usage: $0 <fuzz_target_binary>" - exit 1 -fi - -main $1 -exit $? diff --git a/infra/base-images/base-runner/test_one.py b/infra/base-images/base-runner/test_one.py new file mode 100755 index 000000000..9bdb75faf --- /dev/null +++ b/infra/base-images/base-runner/test_one.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ +"""Does bad_build_check on a fuzz target in $OUT.""" +import os +import sys + +import test_all + + +def test_one(fuzz_target): + """Does bad_build_check on one fuzz target. Returns True on success.""" + with test_all.use_different_out_dir(): + fuzz_target_path = os.path.join(os.environ['OUT'], fuzz_target) + return test_all.do_bad_build_check(fuzz_target_path).returncode == 0 + + +def main(): + """Does bad_build_check on one fuzz target. Returns 1 on failure, 0 on + success.""" + if len(sys.argv) != 2: + print('Usage: %d <fuzz_target>', sys.argv[0]) + return 1 + + fuzz_target_binary = sys.argv[1] + return 0 if test_one(fuzz_target_binary) else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/infra/base-images/base-sanitizer-libs-builder/msan_build.py b/infra/base-images/base-sanitizer-libs-builder/msan_build.py index 928b1a596..5ea00ab10 100755 --- a/infra/base-images/base-sanitizer-libs-builder/msan_build.py +++ b/infra/base-images/base-sanitizer-libs-builder/msan_build.py @@ -73,7 +73,9 @@ def SetUpEnvironment(work_dir): dpkg_host_architecture = wrapper_utils.DpkgHostArchitecture() wrapper_utils.CreateSymlinks( - compiler_wrapper_path, bin_dir, [ + compiler_wrapper_path, + bin_dir, + [ 'clang', 'clang++', # Not all build rules respect $CC/$CXX, so make additional symlinks. @@ -101,41 +103,35 @@ def SetUpEnvironment(work_dir): env['DPKG_GENSYMBOLS_CHECK_LEVEL'] = '0' # debian/rules can set DPKG_GENSYMBOLS_CHECK_LEVEL explicitly, so override it. - gen_symbols_wrapper = ( - '#!/bin/sh\n' - 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n' - '/usr/bin/dpkg-gensymbols "$@"\n') + gen_symbols_wrapper = ('#!/bin/sh\n' + 'export DPKG_GENSYMBOLS_CHECK_LEVEL=0\n' + '/usr/bin/dpkg-gensymbols "$@"\n') - wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols', - gen_symbols_wrapper) + wrapper_utils.InstallWrapper(bin_dir, 'dpkg-gensymbols', gen_symbols_wrapper) # Install no-op strip binaries. - no_op_strip = ('#!/bin/sh\n' - 'exit 0\n') - wrapper_utils.InstallWrapper( - bin_dir, 'strip', no_op_strip, - [dpkg_host_architecture + '-strip']) + no_op_strip = ('#!/bin/sh\n' 'exit 0\n') + wrapper_utils.InstallWrapper(bin_dir, 'strip', no_op_strip, + [dpkg_host_architecture + '-strip']) env['PATH'] = bin_dir + ':' + os.environ['PATH'] # nocheck doesn't disable override_dh_auto_test. So we have this hack to try # to disable "make check" or "make test" invocations. - make_wrapper = ( - '#!/bin/bash\n' - 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n' - ' exit 0\n' - 'fi\n' - '/usr/bin/make "$@"\n') - wrapper_utils.InstallWrapper(bin_dir, 'make', - make_wrapper) + make_wrapper = ('#!/bin/bash\n' + 'if [ "$1" = "test" ] || [ "$1" = "check" ]; then\n' + ' exit 0\n' + 'fi\n' + '/usr/bin/make "$@"\n') + wrapper_utils.InstallWrapper(bin_dir, 'make', make_wrapper) # Prevent entire build from failing because of bugs/uninstrumented in tools # that are part of the build. msan_log_dir = os.path.join(work_dir, 'msan') os.mkdir(msan_log_dir) msan_log_path = os.path.join(msan_log_dir, 'log') - env['MSAN_OPTIONS'] = ( - 'halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + msan_log_path) + env['MSAN_OPTIONS'] = ('halt_on_error=0:exitcode=0:report_umrs=0:log_path=' + + msan_log_path) # Increase maximum stack size to prevent tests from failing. limit = 128 * 1024 * 1024 @@ -207,7 +203,7 @@ def ExtractLibraries(deb_paths, work_directory, output_directory): target_file_path = os.path.join(output_directory, rel_file_path) extracted.append(target_file_path) - + if os.path.lexists(target_file_path): os.remove(target_file_path) @@ -215,8 +211,8 @@ def ExtractLibraries(deb_paths, work_directory, output_directory): link_path = os.readlink(file_path) if os.path.isabs(link_path): # Make absolute links relative. - link_path = os.path.relpath( - link_path, os.path.join('/', rel_directory)) + link_path = os.path.relpath(link_path, + os.path.join('/', rel_directory)) os.symlink(link_path, target_file_path) else: @@ -244,8 +240,8 @@ def GetPackage(package_name): def PatchRpath(path, output_directory): """Patch rpath to be relative to $ORIGIN.""" try: - rpaths = subprocess.check_output( - ['patchelf', '--print-rpath', path]).strip() + rpaths = subprocess.check_output(['patchelf', '--print-rpath', + path]).strip() except subprocess.CalledProcessError: return @@ -262,15 +258,13 @@ def PatchRpath(path, output_directory): processed_rpath.append(rpath) continue - processed_rpath.append(os.path.join( - '$ORIGIN', - os.path.relpath(rpath, rel_directory))) + processed_rpath.append( + os.path.join('$ORIGIN', os.path.relpath(rpath, rel_directory))) processed_rpath = ':'.join(processed_rpath) print('Patching rpath for', path, 'to', processed_rpath) subprocess.check_call( - ['patchelf', '--force-rpath', '--set-rpath', - processed_rpath, path]) + ['patchelf', '--force-rpath', '--set-rpath', processed_rpath, path]) def _CollectDependencies(apt_cache, pkg, cache, dependencies): @@ -331,7 +325,11 @@ def GetBuildList(package_name): class MSanBuilder(object): """MSan builder.""" - def __init__(self, debug=False, log_path=None, work_dir=None, no_track_origins=False): + def __init__(self, + debug=False, + log_path=None, + work_dir=None, + no_track_origins=False): self.debug = debug self.log_path = log_path self.work_dir = work_dir @@ -396,19 +394,24 @@ class MSanBuilder(object): extracted_paths = ExtractLibraries(deb_paths, self.work_dir, extract_directory) for extracted_path in extracted_paths: - if not os.path.islink(extracted_path): - PatchRpath(extracted_path, extract_directory) + if os.path.islink(extracted_path): + continue + if os.path.basename(extracted_path) == 'llvm-symbolizer': + continue + PatchRpath(extracted_path, extract_directory) def main(): parser = argparse.ArgumentParser('msan_build.py', description='MSan builder.') parser.add_argument('package_names', nargs='+', help='Name of the packages.') parser.add_argument('output_dir', help='Output directory.') - parser.add_argument('--create-subdirs', action='store_true', + parser.add_argument('--create-subdirs', + action='store_true', help=('Create subdirectories in the output ' 'directory for each package.')) parser.add_argument('--work-dir', help='Work directory.') - parser.add_argument('--no-build-deps', action='store_true', + parser.add_argument('--no-build-deps', + action='store_true', help='Don\'t build dependencies.') parser.add_argument('--debug', action='store_true', help='Enable debug mode.') parser.add_argument('--log-path', help='Log path for debugging.') @@ -445,7 +448,8 @@ def main(): for package_name in package_names: print('\t', package_name) - with MSanBuilder(debug=args.debug, log_path=args.log_path, + with MSanBuilder(debug=args.debug, + log_path=args.log_path, work_dir=args.work_dir, no_track_origins=args.no_track_origins) as builder: for package_name in package_names: |