diff options
Diffstat (limited to 'infra/base-images/base-builder')
-rw-r--r-- | infra/base-images/base-builder/Dockerfile | 91 | ||||
-rwxr-xr-x | infra/base-images/base-builder/bazel_build_fuzz_tests | 12 | ||||
-rw-r--r-- | infra/base-images/base-builder/bisect_clang_test.py | 18 | ||||
-rwxr-xr-x | infra/base-images/base-builder/cargo | 8 | ||||
-rwxr-xr-x | infra/base-images/base-builder/compile | 50 | ||||
-rw-r--r-- | infra/base-images/base-builder/compile_afl | 25 | ||||
-rwxr-xr-x | infra/base-images/base-builder/compile_go_fuzzer | 6 | ||||
-rwxr-xr-x | infra/base-images/base-builder/install_go.sh | 28 | ||||
-rwxr-xr-x | infra/base-images/base-builder/install_java.sh | 37 | ||||
-rwxr-xr-x | infra/base-images/base-builder/install_python.sh | 21 | ||||
-rwxr-xr-x | infra/base-images/base-builder/install_rust.sh | 21 | ||||
-rwxr-xr-x | infra/base-images/base-builder/install_swift.sh | 66 | ||||
-rw-r--r-- | infra/base-images/base-builder/llvmsymbol.diff | 50 | ||||
-rwxr-xr-x | infra/base-images/base-builder/write_labels.py | 2 |
14 files changed, 330 insertions, 105 deletions
diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile index d802f247a..256e7be56 100644 --- a/infra/base-images/base-builder/Dockerfile +++ b/infra/base-images/base-builder/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2016 Google Inc. +# Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -61,71 +61,15 @@ RUN export PYTHON_DEPS="\ rm -rf /usr/local/lib/python3.8/test && \ apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888 -# Install latest atheris for python fuzzing, pyinstaller for fuzzer packaging, -# six for Bazel rules. +# Install six for Bazel rules. RUN unset CFLAGS CXXFLAGS && pip3 install -v --no-cache-dir \ - atheris pyinstaller==4.1 six==1.15.0 && \ - rm -rf /tmp/* - -# Download and install the latest stable Go. -RUN cd /tmp && \ - curl -O https://storage.googleapis.com/golang/getgo/installer_linux && \ - chmod +x ./installer_linux && \ - SHELL="bash" ./installer_linux && \ - rm -rf ./installer_linux - -# Set up Golang environment variables (copied from /root/.bash_profile). -ENV GOPATH /root/go - -# /root/.go/bin is for the standard Go binaries (i.e. go, gofmt, etc). -# $GOPATH/bin is for the binaries from the dependencies installed via "go get". -ENV PATH $PATH:/root/.go/bin:$GOPATH/bin - -# Uses golang 1.14+ cmd/compile's native libfuzzer instrumentation. -RUN go get -u github.com/mdempsky/go114-fuzz-build && \ - ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz - -# Install Rust and cargo-fuzz for libFuzzer instrumentation. -ENV CARGO_HOME=/rust -ENV RUSTUP_HOME=/rust/rustup -ENV PATH=$PATH:/rust/bin -RUN curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal -RUN cargo install cargo-fuzz && rm -rf /rust/registry -# Needed to recompile rust std library for MSAN -RUN rustup component add rust-src --toolchain nightly -# Set up custom environment variable for source code copy for coverage reports -ENV OSSFUZZ_RUSTPATH /rust + six==1.15.0 && rm -rf /tmp/* # Install Bazel through Bazelisk, which automatically fetches the latest Bazel version. -ENV BAZELISK_VERSION 1.7.4 +ENV BAZELISK_VERSION 1.9.0 RUN curl -L https://github.com/bazelbuild/bazelisk/releases/download/v$BAZELISK_VERSION/bazelisk-linux-amd64 -o /usr/local/bin/bazel && \ chmod +x /usr/local/bin/bazel -# Install OpenJDK 15 and trim its size by removing unused components. -ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64 -ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server -ENV PATH=$PATH:$JAVA_HOME/bin -RUN cd /tmp && \ - curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \ - mkdir -p $JAVA_HOME && \ - tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ - rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \ - rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip - -# Install the latest Jazzer in $OUT. -# jazzer_api_deploy.jar is required only at build-time, the agent and the -# drivers are copied to $OUT as they need to be present on the runners. -ENV JAZZER_API_PATH "/usr/local/lib/jazzer_api_deploy.jar" -RUN cd $SRC/ && \ - git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \ - cd jazzer && \ - bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \ - //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //agent:jazzer_api_deploy.jar && \ - cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan /usr/local/bin/ && \ - cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH && \ - rm -rf ~/.cache/bazel ~/.cache/bazelisk && \ - rm -rf $SRC/jazzer - # Default build flags for various sanitizers. ENV SANITIZER_FLAGS_address "-fsanitize=address -fsanitize-address-use-after-scope" @@ -181,7 +125,7 @@ WORKDIR $SRC # TODO: switch to -b stable once we can. RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \ cd aflplusplus && \ - git checkout 2102264acf5c271b7560a82771b3af8136af9354 + git checkout 4fe572b80f76ff0b0e916b639d1e04d5af48b157 RUN cd $SRC && \ curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \ @@ -190,14 +134,25 @@ RUN cd $SRC && \ tar -xzv --strip-components=1 -f $SRC/oss-fuzz.tar.gz && \ rm -rf examples $SRC/oss-fuzz.tar.gz -COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ - compile_go_fuzzer precompile_honggfuzz precompile_afl debug_afl srcmap \ - write_labels.py bazel_build_fuzz_tests /usr/local/bin/ - -COPY detect_repo.py /opt/cifuzz/ -COPY ossfuzz_coverage_runner.go $GOPATH +# Do precompiles before copying other scripts for better cache efficiency. +COPY precompile_afl /usr/local/bin/ +RUN precompile_afl +COPY precompile_honggfuzz /usr/local/bin/ RUN precompile_honggfuzz -RUN precompile_afl + +COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \ + compile_go_fuzzer debug_afl srcmap \ + write_labels.py bazel_build_fuzz_tests \ + # Go, java, and swift installation scripts. + install_go.sh \ + install_java.sh \ + install_python.sh \ + install_rust.sh \ + install_swift.sh \ + /usr/local/bin/ + +COPY llvmsymbol.diff $SRC +COPY detect_repo.py /opt/cifuzz/ CMD ["compile"] diff --git a/infra/base-images/base-builder/bazel_build_fuzz_tests b/infra/base-images/base-builder/bazel_build_fuzz_tests index 86740ee01..dca79f3f2 100755 --- a/infra/base-images/base-builder/bazel_build_fuzz_tests +++ b/infra/base-images/base-builder/bazel_build_fuzz_tests @@ -22,10 +22,17 @@ : "${BAZEL_TOOL:=bazel}" : "${BAZEL_EXTRA_BUILD_FLAGS:=}" +if [ "$FUZZING_LANGUAGE" = "jvm" ]; then + BAZEL_LANGUAGE=java +else + BAZEL_LANGUAGE=cc +fi + if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then BAZEL_FUZZ_TEST_QUERY=" let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in - \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests) + let lang_fuzz_tests = attr(generator_function, \"^${BAZEL_LANGUAGE}_fuzz_test\$\", \$all_fuzz_tests) in + \$lang_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$lang_fuzz_tests) " fi @@ -42,9 +49,10 @@ done declare -r BAZEL_BUILD_FLAGS=( "-c" "opt" - "--//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \ + "--@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \ "--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \ "--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \ + "--cxxopt=-stdlib=libc++" \ "--linkopt=-lc++" \ "--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \ ${BAZEL_EXTRA_BUILD_FLAGS[*]} diff --git a/infra/base-images/base-builder/bisect_clang_test.py b/infra/base-images/base-builder/bisect_clang_test.py index edf13e759..a11bf8640 100644 --- a/infra/base-images/base-builder/bisect_clang_test.py +++ b/infra/base-images/base-builder/bisect_clang_test.py @@ -127,7 +127,7 @@ def create_mock_popen( return MockPopen -def mock_prepare_build(llvm_project_path): # pylint: disable=unused-argument +def mock_prepare_build_impl(llvm_project_path): # pylint: disable=unused-argument """Mocked prepare_build function.""" return '/work/llvm-build' @@ -138,7 +138,7 @@ class BuildClangTest(BisectClangTestMixin, unittest.TestCase): def test_build_clang_test(self): """Tests that build_clang works as intended.""" with mock.patch('subprocess.Popen', create_mock_popen()) as mock_popen: - with mock.patch('bisect_clang.prepare_build', mock_prepare_build): + with mock.patch('bisect_clang.prepare_build', mock_prepare_build_impl): llvm_src_dir = '/src/llvm-project' bisect_clang.build_clang(llvm_src_dir) self.assertEqual([['ninja', '-C', '/work/llvm-build', 'install']], @@ -170,13 +170,13 @@ class GitRepoTest(BisectClangTestMixin, unittest.TestCase): """Tests test_start_commit works as intended when the test returns an unexpected value.""" - def mock_execute(command, *args, **kwargs): # pylint: disable=unused-argument + def mock_execute_impl(command, *args, **kwargs): # pylint: disable=unused-argument if command == self.test_command: return returncode, '', '' return 0, '', '' - with mock.patch('bisect_clang.execute', mock_execute): - with mock.patch('bisect_clang.prepare_build', mock_prepare_build): + with mock.patch('bisect_clang.execute', mock_execute_impl): + with mock.patch('bisect_clang.prepare_build', mock_prepare_build_impl): with self.assertRaises(bisect_clang.BisectError): self.git.test_start_commit(commit, label, self.test_command) @@ -202,13 +202,13 @@ class GitRepoTest(BisectClangTestMixin, unittest.TestCase): expected value.""" command_args = [] - def mock_execute(command, *args, **kwargs): # pylint: disable=unused-argument + def mock_execute_impl(command, *args, **kwargs): # pylint: disable=unused-argument command_args.append(command) if command == self.test_command: return returncode, '', '' return 0, '', '' - with mock.patch('bisect_clang.execute', mock_execute): + with mock.patch('bisect_clang.execute', mock_execute_impl): self.git.test_start_commit(commit, label, self.test_command) self.assertEqual([ get_git_command('checkout', commit), self.test_command, @@ -247,13 +247,13 @@ class GitRepoTest(BisectClangTestMixin, unittest.TestCase): """Test test_commit works as intended.""" command_args = [] - def mock_execute(command, *args, **kwargs): # pylint: disable=unused-argument + def mock_execute_impl(command, *args, **kwargs): # pylint: disable=unused-argument command_args.append(command) if command == self.test_command: return returncode, output, '' return 0, output, '' - with mock.patch('bisect_clang.execute', mock_execute): + with mock.patch('bisect_clang.execute', mock_execute_impl): result = self.git.test_commit(self.test_command) self.assertEqual([self.test_command, get_git_command('bisect', label)], command_args) diff --git a/infra/base-images/base-builder/cargo b/infra/base-images/base-builder/cargo index bed8e7660..c60c7611b 100755 --- a/infra/base-images/base-builder/cargo +++ b/infra/base-images/base-builder/cargo @@ -27,7 +27,7 @@ then export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix src=$crate_src_abspath/src" fi -if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ] +if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ] && [ $2 = "build" ] then # hack to turn cargo fuzz build into cargo build so as to get coverage # cargo fuzz adds "--target" "x86_64-unknown-linux-gnu" @@ -35,7 +35,11 @@ then # go into fuzz directory if not already the case cd fuzz || true fuzz_src_abspath=`pwd` - export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix fuzz_targets=$fuzz_src_abspath/fuzz_targets" + # Default directory is fuzz_targets, but some projects like image-rs use fuzzers. + while read i; do + export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $i=$fuzz_src_abspath/$i" + # Bash while syntax so that we modify RUSTFLAGS in main shell instead of a subshell. + done <<< "$(ls */*.rs | cut -d/ -f1 | uniq)" # we do not want to trigger debug assertions and stops export RUSTFLAGS="$RUSTFLAGS -C debug-assertions=no" # do not optimize with --release, leading to Malformed instrumentation profile data diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile index 78453c98c..c934d3b5b 100755 --- a/infra/base-images/base-builder/compile +++ b/infra/base-images/base-builder/compile @@ -27,8 +27,8 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then echo "ERROR: JVM projects can be fuzzed with libFuzzer engine only." exit 1 fi - if [ "$SANITIZER" != "address" ]; then - echo "ERROR: JVM projects can be fuzzed with AddressSanitizer only." + if [ "$SANITIZER" != "address" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "undefined" ]; then + echo "ERROR: JVM projects can be fuzzed with AddressSanitizer or UndefinedBehaviorSanitizer only." exit 1 fi if [ "$ARCHITECTURE" != "x86_64" ]; then @@ -43,7 +43,7 @@ if [ "$FUZZING_LANGUAGE" = "python" ]; then exit 1 fi if [ "$SANITIZER" != "address" ] && [ "$SANITIZER" != "undefined" ]; then - echo "ERROR: Python projects can be fuzzed with AddressSanitizer and UndefinedBehaviorSanitizer only." + echo "ERROR: Python projects can be fuzzed with AddressSanitizer or UndefinedBehaviorSanitizer only." exit 1 fi if [ "$ARCHITECTURE" != "x86_64" ]; then @@ -59,7 +59,7 @@ fi if [[ $ARCHITECTURE == "i386" ]]; then export CFLAGS="-m32 $CFLAGS" - cp -R /usr/i386/lib/* /usr/lib + cp -R /usr/i386/lib/* /usr/local/lib fi # JVM projects are fuzzed with Jazzer, which has libFuzzer built in. if [[ $FUZZING_ENGINE != "none" ]] && [[ $FUZZING_LANGUAGE != "jvm" ]]; then @@ -71,15 +71,9 @@ if [[ $SANITIZER_FLAGS = *sanitize=memory* ]] then # Take all libraries from lib/msan and MSAN_LIBS_PATH # export CXXFLAGS_EXTRA="-L/usr/msan/lib $CXXFLAGS_EXTRA" - cp -R /usr/msan/lib/* /usr/lib/ - - if [[ -z "${MSAN_LIBS_PATH-}" ]]; then - echo 'WARNING: Building without MSan instrumented libraries.' - else - # Copy all static libraries only. Don't include .so files because they can - # break non MSan compiled programs. - (cd "$MSAN_LIBS_PATH" && find . -name '*.a' -exec cp --parents '{}' / ';') - fi + cp -R /usr/msan/lib/* /usr/local/lib/ + + echo 'Building without MSan instrumented libraries.' fi # Coverage flag overrides. @@ -118,7 +112,7 @@ export CFLAGS="$CFLAGS $SANITIZER_FLAGS $COVERAGE_FLAGS" export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA" if [ "$FUZZING_LANGUAGE" = "python" ]; then - sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(os.path.dirname(atheris.path()))"` + sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(atheris.path())"` sanitizer_with_fuzzer_output_lib=$OUT/sanitizer_with_fuzzer.so if [ "$SANITIZER" = "address" ]; then cp $sanitizer_with_fuzzer_lib_dir/asan_with_fuzzer.so $sanitizer_with_fuzzer_output_lib @@ -136,7 +130,20 @@ cp $(which llvm-symbolizer) $OUT/ # Copy Jazzer to $OUT if needed. if [ "$FUZZING_LANGUAGE" = "jvm" ]; then - cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $(which jazzer_driver_asan) $OUT/ + cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $OUT/ + jazzer_driver_with_sanitizer=$OUT/jazzer_driver_with_sanitizer + if [ "$SANITIZER" = "address" ]; then + cp $(which jazzer_driver_asan) $jazzer_driver_with_sanitizer + elif [ "$SANITIZER" = "undefined" ]; then + cp $(which jazzer_driver_ubsan) $jazzer_driver_with_sanitizer + elif [ "$SANITIZER" = "coverage" ]; then + # Coverage builds require no instrumentation. + cp $(which jazzer_driver) $jazzer_driver_with_sanitizer + fi + + # Disable leak checking since the JVM triggers too many false positives. + export CFLAGS="$CFLAGS -fno-sanitize=leak" + export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak" fi echo "---------------------------------------------------------------" @@ -144,13 +151,24 @@ echo "CC=$CC" echo "CXX=$CXX" echo "CFLAGS=$CFLAGS" echo "CXXFLAGS=$CXXFLAGS" +echo "RUSTFLAGS=$RUSTFLAGS" echo "---------------------------------------------------------------" BUILD_CMD="bash -eux $SRC/build.sh" +# Set +u temporarily to continue even if GOPATH and OSSFUZZ_RUSTPATH are undefined. +set +u # We need to preserve source code files for generating a code coverage report. # We need exact files that were compiled, so copy both $SRC and $WORK dirs. -COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH $OUT" +COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH /rustc $OUT" +set -u + +if [ "$FUZZING_LANGUAGE" = "rust" ]; then + # Copy rust std lib to its path with a hash. + export rustch=`rustc --version --verbose | grep commit-hash | cut -d' ' -f2` + mkdir -p /rustc/$rustch/ + cp -r /rust/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/ /rustc/$rustch/ +fi if [ "${BUILD_UID-0}" -ne "0" ]; then adduser -u $BUILD_UID --disabled-password --gecos '' builder diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl index dc6624459..d6509c74c 100644 --- a/infra/base-images/base-builder/compile_afl +++ b/infra/base-images/base-builder/compile_afl @@ -22,6 +22,8 @@ # AFL++ settings. export AFL_LLVM_MODE_WORKAROUND=0 export AFL_ENABLE_DICTIONARY=0 +export AFL_ENABLE_CMPLOG=1 +export AFL_LAF_CHANCE=3 # Start compiling afl++. echo "Copying precompiled afl++" @@ -45,19 +47,32 @@ export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_ # AFL compile option roulette. It is OK if they all happen together. -# 40% chance to perform CMPLOG +# 20% chance for CTX-2 coverage instrumentation (Caller conTeXt sensitive +# edge coverage). +test $(($RANDOM % 100)) -lt 20 && { + export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2 + export AFL_ENABLE_CMPLOG=0 + export AFL_LAF_CHANCE=30 +} + +# 40% chance to create a dictionary. +test $(($RANDOM % 100)) -lt 40 && { + export AFL_ENABLE_DICTIONARY=1 +} + +# 60% chance to perform CMPLOG/REDQUEEN. rm -f "$OUT/afl_cmplog.txt" -test $(($RANDOM % 10)) -lt 4 && { +test "$AFL_ENABLE_CMPLOG" = "1" -a $(($RANDOM % 100)) -lt 60 && { export AFL_LLVM_CMPLOG=1 touch "$OUT/afl_cmplog.txt" } -# 10% chance to perform LAF_INTEL -test $(($RANDOM % 10)) -lt 1 && { +# 3% chance to perform COMPCOV/LAF_INTEL. +test $(($RANDOM % 100)) -lt $AFL_LAF_CHANCE && { export AFL_LLVM_LAF_ALL=1 } -# If the targets wants a dictionary - then create one. +# Create a dictionary if one is wanted. test "$AFL_ENABLE_DICTIONARY" = "1" && { export AFL_LLVM_DICT2FILE="$OUT/afl++.dict" } diff --git a/infra/base-images/base-builder/compile_go_fuzzer b/infra/base-images/base-builder/compile_go_fuzzer index 2342800fb..dd8c9f6a1 100755 --- a/infra/base-images/base-builder/compile_go_fuzzer +++ b/infra/base-images/base-builder/compile_go_fuzzer @@ -29,7 +29,7 @@ cd $GOPATH/src/$path || true # in the case we are in the right directory, with go.mod but no go.sum go mod tidy || true # project was downloaded with go get if go list fails -go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; } +go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; } || cd - # project does not have go.mod if go list fails again go list $tags $path || { go mod init $path && go mod tidy ;} @@ -42,7 +42,9 @@ if [[ $SANITIZER = *coverage* ]]; then sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go - fuzzed_repo=`echo $path | cut -d/ -f-3` + # The repo is the module path/name, which is already created above in case it doesn't exist, + # but not always the same as the module path. This is necessary to handle SIV properly. + fuzzed_repo=$(go list $tags -f {{.Module}} "$path") abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo` # give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath diff --git a/infra/base-images/base-builder/install_go.sh b/infra/base-images/base-builder/install_go.sh new file mode 100755 index 000000000..21138831c --- /dev/null +++ b/infra/base-images/base-builder/install_go.sh @@ -0,0 +1,28 @@ +#!/bin/bash -eux +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +cd /tmp +curl -O https://storage.googleapis.com/golang/getgo/installer_linux +chmod +x ./installer_linux +SHELL="bash" ./installer_linux +rm -rf ./installer_linux + +echo 'Set "GOPATH=/root/go"' +echo 'Set "PATH=$PATH:/root/.go/bin:$GOPATH/bin"' + +go get -u github.com/mdempsky/go114-fuzz-build +ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz diff --git a/infra/base-images/base-builder/install_java.sh b/infra/base-images/base-builder/install_java.sh new file mode 100755 index 000000000..560c816a9 --- /dev/null +++ b/infra/base-images/base-builder/install_java.sh @@ -0,0 +1,37 @@ +#!/bin/bash -eux +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +# Install OpenJDK 15 and trim its size by removing unused components. +cd /tmp +curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \ +mkdir -p $JAVA_HOME +tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ +rm -f openjdk-15.0.2_linux-x64_bin.tar.gz +rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip + +# Install the latest Jazzer in $OUT. +# jazzer_api_deploy.jar is required only at build-time, the agent and the +# drivers are copied to $OUT as they need to be present on the runners. +cd $SRC/ +git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \ +cd jazzer +bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \ + //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //driver:jazzer_driver_ubsan //agent:jazzer_api_deploy.jar +cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan bazel-bin/driver/jazzer_driver_ubsan /usr/local/bin/ +cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH +rm -rf ~/.cache/bazel ~/.cache/bazelisk +rm -rf $SRC/jazzer diff --git a/infra/base-images/base-builder/install_python.sh b/infra/base-images/base-builder/install_python.sh new file mode 100755 index 000000000..b9c9a38c3 --- /dev/null +++ b/infra/base-images/base-builder/install_python.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eux +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +echo "ATHERIS INSTALL" +unset CFLAGS CXXFLAGS +pip3 install -v --no-cache-dir atheris>=2.0.6 pyinstaller==4.1 +rm -rf /tmp/* diff --git a/infra/base-images/base-builder/install_rust.sh b/infra/base-images/base-builder/install_rust.sh new file mode 100755 index 000000000..cbb461fd6 --- /dev/null +++ b/infra/base-images/base-builder/install_rust.sh @@ -0,0 +1,21 @@ +#!/bin/bash -eux +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal +cargo install cargo-fuzz && rm -rf /rust/registry +# Needed to recompile rust std library for MSAN +rustup component add rust-src --toolchain nightly diff --git a/infra/base-images/base-builder/install_swift.sh b/infra/base-images/base-builder/install_swift.sh new file mode 100755 index 000000000..d88a7b5cd --- /dev/null +++ b/infra/base-images/base-builder/install_swift.sh @@ -0,0 +1,66 @@ +#!/bin/bash -eux +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + + +SWIFT_PACKAGES="wget \ + binutils \ + git \ + gnupg2 \ + libc6-dev \ + libcurl4 \ + libedit2 \ + libgcc-9-dev \ + libpython2.7 \ + libsqlite3-0 \ + libstdc++-9-dev \ + libxml2 \ + libz3-dev \ + pkg-config \ + tzdata \ + zlib1g-dev" +SWIFT_SYMBOLIZER_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev" +apt-get update && apt install -y $SWIFT_PACKAGES && \ + apt install -y $SWIFT_SYMBOLIZER_PACKAGES --no-install-recommends + + +wget https://swift.org/builds/swift-5.4.2-release/ubuntu2004/swift-5.4.2-RELEASE/swift-5.4.2-RELEASE-ubuntu20.04.tar.gz +tar xzf swift-5.4.2-RELEASE-ubuntu20.04.tar.gz +cp -r swift-5.4.2-RELEASE-ubuntu20.04/usr/* /usr/ +rm -rf swift-5.4.2-RELEASE-ubuntu20.04.tar.gz +# TODO: Move to a seperate work dir +git clone --depth 1 https://github.com/llvm/llvm-project.git +cd llvm-project +git apply ../llvmsymbol.diff --verbose +cmake -G "Ninja" \ + -DLIBCXX_ENABLE_SHARED=OFF \ + -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ + -DLIBCXXABI_ENABLE_SHARED=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD=X86 \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DLLVM_BUILD_TESTS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF llvm +ninja -j$(nproc) llvm-symbolizer +cp bin/llvm-symbolizer /usr/local/bin/llvm-symbolizer-swift + +cd $SRC +rm -rf llvm-project llvmsymbol.diff + +# TODO: Cleanup packages +apt-get remove --purge -y wget zlib1g-dev +apt-get autoremove -y diff --git a/infra/base-images/base-builder/llvmsymbol.diff b/infra/base-images/base-builder/llvmsymbol.diff new file mode 100644 index 000000000..70181bf39 --- /dev/null +++ b/infra/base-images/base-builder/llvmsymbol.diff @@ -0,0 +1,50 @@ +diff --git a/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt b/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt +index acfb3bd0e..a499ee2e0 100644 +--- a/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt ++++ b/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt +@@ -12,4 +12,11 @@ add_llvm_component_library(LLVMSymbolize + Object + Support + Demangle +- ) ++ ++ LINK_LIBS ++ /usr/lib/swift_static/linux/libswiftCore.a ++ /usr/lib/swift_static/linux/libicui18nswift.a ++ /usr/lib/swift_static/linux/libicuucswift.a ++ /usr/lib/swift_static/linux/libicudataswift.a ++ /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ++) +diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp +index fb4875f79..0030769ee 100644 +--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp ++++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp +@@ -36,6 +36,13 @@ + #include <cassert> + #include <cstring> + ++ ++extern "C" char *swift_demangle(const char *mangledName, ++ size_t mangledNameLength, ++ char *outputBuffer, ++ size_t *outputBufferSize, ++ uint32_t flags); ++ + namespace llvm { + namespace symbolize { + +@@ -678,6 +685,14 @@ LLVMSymbolizer::DemangleName(const std::string &Name, + free(DemangledName); + return Result; + } ++ if (!Name.empty() && Name.front() == '$') { ++ char *DemangledName = swift_demangle(Name.c_str(), Name.length(), 0, 0, 0); ++ if (DemangledName) { ++ std::string Result = DemangledName; ++ free(DemangledName); ++ return Result; ++ } ++ } + + if (DbiModuleDescriptor && DbiModuleDescriptor->isWin32Module()) + return std::string(demanglePE32ExternCFunc(Name)); diff --git a/infra/base-images/base-builder/write_labels.py b/infra/base-images/base-builder/write_labels.py index 6766e37fe..92a820a43 100755 --- a/infra/base-images/base-builder/write_labels.py +++ b/infra/base-images/base-builder/write_labels.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 # Copyright 2021 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); |