aboutsummaryrefslogtreecommitdiff
path: root/infra/base-images/base-builder
diff options
context:
space:
mode:
Diffstat (limited to 'infra/base-images/base-builder')
-rw-r--r--infra/base-images/base-builder/Dockerfile91
-rwxr-xr-xinfra/base-images/base-builder/bazel_build_fuzz_tests12
-rw-r--r--infra/base-images/base-builder/bisect_clang_test.py18
-rwxr-xr-xinfra/base-images/base-builder/cargo8
-rwxr-xr-xinfra/base-images/base-builder/compile50
-rw-r--r--infra/base-images/base-builder/compile_afl25
-rwxr-xr-xinfra/base-images/base-builder/compile_go_fuzzer6
-rwxr-xr-xinfra/base-images/base-builder/install_go.sh28
-rwxr-xr-xinfra/base-images/base-builder/install_java.sh37
-rwxr-xr-xinfra/base-images/base-builder/install_python.sh21
-rwxr-xr-xinfra/base-images/base-builder/install_rust.sh21
-rwxr-xr-xinfra/base-images/base-builder/install_swift.sh66
-rw-r--r--infra/base-images/base-builder/llvmsymbol.diff50
-rwxr-xr-xinfra/base-images/base-builder/write_labels.py2
14 files changed, 330 insertions, 105 deletions
diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile
index d802f247a..256e7be56 100644
--- a/infra/base-images/base-builder/Dockerfile
+++ b/infra/base-images/base-builder/Dockerfile
@@ -1,4 +1,4 @@
-# Copyright 2016 Google Inc.
+# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -61,71 +61,15 @@ RUN export PYTHON_DEPS="\
rm -rf /usr/local/lib/python3.8/test && \
apt-get remove -y $PYTHON_DEPS # https://github.com/google/oss-fuzz/issues/3888
-# Install latest atheris for python fuzzing, pyinstaller for fuzzer packaging,
-# six for Bazel rules.
+# Install six for Bazel rules.
RUN unset CFLAGS CXXFLAGS && pip3 install -v --no-cache-dir \
- atheris pyinstaller==4.1 six==1.15.0 && \
- rm -rf /tmp/*
-
-# Download and install the latest stable Go.
-RUN cd /tmp && \
- curl -O https://storage.googleapis.com/golang/getgo/installer_linux && \
- chmod +x ./installer_linux && \
- SHELL="bash" ./installer_linux && \
- rm -rf ./installer_linux
-
-# Set up Golang environment variables (copied from /root/.bash_profile).
-ENV GOPATH /root/go
-
-# /root/.go/bin is for the standard Go binaries (i.e. go, gofmt, etc).
-# $GOPATH/bin is for the binaries from the dependencies installed via "go get".
-ENV PATH $PATH:/root/.go/bin:$GOPATH/bin
-
-# Uses golang 1.14+ cmd/compile's native libfuzzer instrumentation.
-RUN go get -u github.com/mdempsky/go114-fuzz-build && \
- ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz
-
-# Install Rust and cargo-fuzz for libFuzzer instrumentation.
-ENV CARGO_HOME=/rust
-ENV RUSTUP_HOME=/rust/rustup
-ENV PATH=$PATH:/rust/bin
-RUN curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal
-RUN cargo install cargo-fuzz && rm -rf /rust/registry
-# Needed to recompile rust std library for MSAN
-RUN rustup component add rust-src --toolchain nightly
-# Set up custom environment variable for source code copy for coverage reports
-ENV OSSFUZZ_RUSTPATH /rust
+ six==1.15.0 && rm -rf /tmp/*
# Install Bazel through Bazelisk, which automatically fetches the latest Bazel version.
-ENV BAZELISK_VERSION 1.7.4
+ENV BAZELISK_VERSION 1.9.0
RUN curl -L https://github.com/bazelbuild/bazelisk/releases/download/v$BAZELISK_VERSION/bazelisk-linux-amd64 -o /usr/local/bin/bazel && \
chmod +x /usr/local/bin/bazel
-# Install OpenJDK 15 and trim its size by removing unused components.
-ENV JAVA_HOME=/usr/lib/jvm/java-15-openjdk-amd64
-ENV JVM_LD_LIBRARY_PATH=$JAVA_HOME/lib/server
-ENV PATH=$PATH:$JAVA_HOME/bin
-RUN cd /tmp && \
- curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \
- mkdir -p $JAVA_HOME && \
- tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \
- rm -f openjdk-15.0.2_linux-x64_bin.tar.gz && \
- rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip
-
-# Install the latest Jazzer in $OUT.
-# jazzer_api_deploy.jar is required only at build-time, the agent and the
-# drivers are copied to $OUT as they need to be present on the runners.
-ENV JAZZER_API_PATH "/usr/local/lib/jazzer_api_deploy.jar"
-RUN cd $SRC/ && \
- git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \
- cd jazzer && \
- bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \
- //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //agent:jazzer_api_deploy.jar && \
- cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan /usr/local/bin/ && \
- cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH && \
- rm -rf ~/.cache/bazel ~/.cache/bazelisk && \
- rm -rf $SRC/jazzer
-
# Default build flags for various sanitizers.
ENV SANITIZER_FLAGS_address "-fsanitize=address -fsanitize-address-use-after-scope"
@@ -181,7 +125,7 @@ WORKDIR $SRC
# TODO: switch to -b stable once we can.
RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \
cd aflplusplus && \
- git checkout 2102264acf5c271b7560a82771b3af8136af9354
+ git checkout 4fe572b80f76ff0b0e916b639d1e04d5af48b157
RUN cd $SRC && \
curl -L -O https://github.com/google/honggfuzz/archive/oss-fuzz.tar.gz && \
@@ -190,14 +134,25 @@ RUN cd $SRC && \
tar -xzv --strip-components=1 -f $SRC/oss-fuzz.tar.gz && \
rm -rf examples $SRC/oss-fuzz.tar.gz
-COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
- compile_go_fuzzer precompile_honggfuzz precompile_afl debug_afl srcmap \
- write_labels.py bazel_build_fuzz_tests /usr/local/bin/
-
-COPY detect_repo.py /opt/cifuzz/
-COPY ossfuzz_coverage_runner.go $GOPATH
+# Do precompiles before copying other scripts for better cache efficiency.
+COPY precompile_afl /usr/local/bin/
+RUN precompile_afl
+COPY precompile_honggfuzz /usr/local/bin/
RUN precompile_honggfuzz
-RUN precompile_afl
+
+COPY cargo compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
+ compile_go_fuzzer debug_afl srcmap \
+ write_labels.py bazel_build_fuzz_tests \
+ # Go, java, and swift installation scripts.
+ install_go.sh \
+ install_java.sh \
+ install_python.sh \
+ install_rust.sh \
+ install_swift.sh \
+ /usr/local/bin/
+
+COPY llvmsymbol.diff $SRC
+COPY detect_repo.py /opt/cifuzz/
CMD ["compile"]
diff --git a/infra/base-images/base-builder/bazel_build_fuzz_tests b/infra/base-images/base-builder/bazel_build_fuzz_tests
index 86740ee01..dca79f3f2 100755
--- a/infra/base-images/base-builder/bazel_build_fuzz_tests
+++ b/infra/base-images/base-builder/bazel_build_fuzz_tests
@@ -22,10 +22,17 @@
: "${BAZEL_TOOL:=bazel}"
: "${BAZEL_EXTRA_BUILD_FLAGS:=}"
+if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
+ BAZEL_LANGUAGE=java
+else
+ BAZEL_LANGUAGE=cc
+fi
+
if [[ -z "${BAZEL_FUZZ_TEST_QUERY:-}" ]]; then
BAZEL_FUZZ_TEST_QUERY="
let all_fuzz_tests = attr(tags, \"${BAZEL_FUZZ_TEST_TAG}\", \"//...\") in
- \$all_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$all_fuzz_tests)
+ let lang_fuzz_tests = attr(generator_function, \"^${BAZEL_LANGUAGE}_fuzz_test\$\", \$all_fuzz_tests) in
+ \$lang_fuzz_tests - attr(tags, \"${BAZEL_FUZZ_TEST_EXCLUDE_TAG}\", \$lang_fuzz_tests)
"
fi
@@ -42,9 +49,10 @@ done
declare -r BAZEL_BUILD_FLAGS=(
"-c" "opt"
- "--//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \
+ "--@rules_fuzzing//fuzzing:cc_engine=@rules_fuzzing_oss_fuzz//:oss_fuzz_engine" \
"--@rules_fuzzing//fuzzing:cc_engine_instrumentation=oss-fuzz" \
"--@rules_fuzzing//fuzzing:cc_engine_sanitizer=none" \
+ "--cxxopt=-stdlib=libc++" \
"--linkopt=-lc++" \
"--action_env=CC=${CC}" "--action_env=CXX=${CXX}" \
${BAZEL_EXTRA_BUILD_FLAGS[*]}
diff --git a/infra/base-images/base-builder/bisect_clang_test.py b/infra/base-images/base-builder/bisect_clang_test.py
index edf13e759..a11bf8640 100644
--- a/infra/base-images/base-builder/bisect_clang_test.py
+++ b/infra/base-images/base-builder/bisect_clang_test.py
@@ -127,7 +127,7 @@ def create_mock_popen(
return MockPopen
-def mock_prepare_build(llvm_project_path): # pylint: disable=unused-argument
+def mock_prepare_build_impl(llvm_project_path): # pylint: disable=unused-argument
"""Mocked prepare_build function."""
return '/work/llvm-build'
@@ -138,7 +138,7 @@ class BuildClangTest(BisectClangTestMixin, unittest.TestCase):
def test_build_clang_test(self):
"""Tests that build_clang works as intended."""
with mock.patch('subprocess.Popen', create_mock_popen()) as mock_popen:
- with mock.patch('bisect_clang.prepare_build', mock_prepare_build):
+ with mock.patch('bisect_clang.prepare_build', mock_prepare_build_impl):
llvm_src_dir = '/src/llvm-project'
bisect_clang.build_clang(llvm_src_dir)
self.assertEqual([['ninja', '-C', '/work/llvm-build', 'install']],
@@ -170,13 +170,13 @@ class GitRepoTest(BisectClangTestMixin, unittest.TestCase):
"""Tests test_start_commit works as intended when the test returns an
unexpected value."""
- def mock_execute(command, *args, **kwargs): # pylint: disable=unused-argument
+ def mock_execute_impl(command, *args, **kwargs): # pylint: disable=unused-argument
if command == self.test_command:
return returncode, '', ''
return 0, '', ''
- with mock.patch('bisect_clang.execute', mock_execute):
- with mock.patch('bisect_clang.prepare_build', mock_prepare_build):
+ with mock.patch('bisect_clang.execute', mock_execute_impl):
+ with mock.patch('bisect_clang.prepare_build', mock_prepare_build_impl):
with self.assertRaises(bisect_clang.BisectError):
self.git.test_start_commit(commit, label, self.test_command)
@@ -202,13 +202,13 @@ class GitRepoTest(BisectClangTestMixin, unittest.TestCase):
expected value."""
command_args = []
- def mock_execute(command, *args, **kwargs): # pylint: disable=unused-argument
+ def mock_execute_impl(command, *args, **kwargs): # pylint: disable=unused-argument
command_args.append(command)
if command == self.test_command:
return returncode, '', ''
return 0, '', ''
- with mock.patch('bisect_clang.execute', mock_execute):
+ with mock.patch('bisect_clang.execute', mock_execute_impl):
self.git.test_start_commit(commit, label, self.test_command)
self.assertEqual([
get_git_command('checkout', commit), self.test_command,
@@ -247,13 +247,13 @@ class GitRepoTest(BisectClangTestMixin, unittest.TestCase):
"""Test test_commit works as intended."""
command_args = []
- def mock_execute(command, *args, **kwargs): # pylint: disable=unused-argument
+ def mock_execute_impl(command, *args, **kwargs): # pylint: disable=unused-argument
command_args.append(command)
if command == self.test_command:
return returncode, output, ''
return 0, output, ''
- with mock.patch('bisect_clang.execute', mock_execute):
+ with mock.patch('bisect_clang.execute', mock_execute_impl):
result = self.git.test_commit(self.test_command)
self.assertEqual([self.test_command,
get_git_command('bisect', label)], command_args)
diff --git a/infra/base-images/base-builder/cargo b/infra/base-images/base-builder/cargo
index bed8e7660..c60c7611b 100755
--- a/infra/base-images/base-builder/cargo
+++ b/infra/base-images/base-builder/cargo
@@ -27,7 +27,7 @@ then
export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix src=$crate_src_abspath/src"
fi
-if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ]
+if [ "$SANITIZER" = "coverage" ] && [ $1 = "fuzz" ] && [ $2 = "build" ]
then
# hack to turn cargo fuzz build into cargo build so as to get coverage
# cargo fuzz adds "--target" "x86_64-unknown-linux-gnu"
@@ -35,7 +35,11 @@ then
# go into fuzz directory if not already the case
cd fuzz || true
fuzz_src_abspath=`pwd`
- export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix fuzz_targets=$fuzz_src_abspath/fuzz_targets"
+ # Default directory is fuzz_targets, but some projects like image-rs use fuzzers.
+ while read i; do
+ export RUSTFLAGS="$RUSTFLAGS --remap-path-prefix $i=$fuzz_src_abspath/$i"
+ # Bash while syntax so that we modify RUSTFLAGS in main shell instead of a subshell.
+ done <<< "$(ls */*.rs | cut -d/ -f1 | uniq)"
# we do not want to trigger debug assertions and stops
export RUSTFLAGS="$RUSTFLAGS -C debug-assertions=no"
# do not optimize with --release, leading to Malformed instrumentation profile data
diff --git a/infra/base-images/base-builder/compile b/infra/base-images/base-builder/compile
index 78453c98c..c934d3b5b 100755
--- a/infra/base-images/base-builder/compile
+++ b/infra/base-images/base-builder/compile
@@ -27,8 +27,8 @@ if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
echo "ERROR: JVM projects can be fuzzed with libFuzzer engine only."
exit 1
fi
- if [ "$SANITIZER" != "address" ]; then
- echo "ERROR: JVM projects can be fuzzed with AddressSanitizer only."
+ if [ "$SANITIZER" != "address" ] && [ "$SANITIZER" != "coverage" ] && [ "$SANITIZER" != "undefined" ]; then
+ echo "ERROR: JVM projects can be fuzzed with AddressSanitizer or UndefinedBehaviorSanitizer only."
exit 1
fi
if [ "$ARCHITECTURE" != "x86_64" ]; then
@@ -43,7 +43,7 @@ if [ "$FUZZING_LANGUAGE" = "python" ]; then
exit 1
fi
if [ "$SANITIZER" != "address" ] && [ "$SANITIZER" != "undefined" ]; then
- echo "ERROR: Python projects can be fuzzed with AddressSanitizer and UndefinedBehaviorSanitizer only."
+ echo "ERROR: Python projects can be fuzzed with AddressSanitizer or UndefinedBehaviorSanitizer only."
exit 1
fi
if [ "$ARCHITECTURE" != "x86_64" ]; then
@@ -59,7 +59,7 @@ fi
if [[ $ARCHITECTURE == "i386" ]]; then
export CFLAGS="-m32 $CFLAGS"
- cp -R /usr/i386/lib/* /usr/lib
+ cp -R /usr/i386/lib/* /usr/local/lib
fi
# JVM projects are fuzzed with Jazzer, which has libFuzzer built in.
if [[ $FUZZING_ENGINE != "none" ]] && [[ $FUZZING_LANGUAGE != "jvm" ]]; then
@@ -71,15 +71,9 @@ if [[ $SANITIZER_FLAGS = *sanitize=memory* ]]
then
# Take all libraries from lib/msan and MSAN_LIBS_PATH
# export CXXFLAGS_EXTRA="-L/usr/msan/lib $CXXFLAGS_EXTRA"
- cp -R /usr/msan/lib/* /usr/lib/
-
- if [[ -z "${MSAN_LIBS_PATH-}" ]]; then
- echo 'WARNING: Building without MSan instrumented libraries.'
- else
- # Copy all static libraries only. Don't include .so files because they can
- # break non MSan compiled programs.
- (cd "$MSAN_LIBS_PATH" && find . -name '*.a' -exec cp --parents '{}' / ';')
- fi
+ cp -R /usr/msan/lib/* /usr/local/lib/
+
+ echo 'Building without MSan instrumented libraries.'
fi
# Coverage flag overrides.
@@ -118,7 +112,7 @@ export CFLAGS="$CFLAGS $SANITIZER_FLAGS $COVERAGE_FLAGS"
export CXXFLAGS="$CFLAGS $CXXFLAGS_EXTRA"
if [ "$FUZZING_LANGUAGE" = "python" ]; then
- sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(os.path.dirname(atheris.path()))"`
+ sanitizer_with_fuzzer_lib_dir=`python3 -c "import atheris; import os; print(atheris.path())"`
sanitizer_with_fuzzer_output_lib=$OUT/sanitizer_with_fuzzer.so
if [ "$SANITIZER" = "address" ]; then
cp $sanitizer_with_fuzzer_lib_dir/asan_with_fuzzer.so $sanitizer_with_fuzzer_output_lib
@@ -136,7 +130,20 @@ cp $(which llvm-symbolizer) $OUT/
# Copy Jazzer to $OUT if needed.
if [ "$FUZZING_LANGUAGE" = "jvm" ]; then
- cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $(which jazzer_driver_asan) $OUT/
+ cp $(which jazzer_agent_deploy.jar) $(which jazzer_driver) $OUT/
+ jazzer_driver_with_sanitizer=$OUT/jazzer_driver_with_sanitizer
+ if [ "$SANITIZER" = "address" ]; then
+ cp $(which jazzer_driver_asan) $jazzer_driver_with_sanitizer
+ elif [ "$SANITIZER" = "undefined" ]; then
+ cp $(which jazzer_driver_ubsan) $jazzer_driver_with_sanitizer
+ elif [ "$SANITIZER" = "coverage" ]; then
+ # Coverage builds require no instrumentation.
+ cp $(which jazzer_driver) $jazzer_driver_with_sanitizer
+ fi
+
+ # Disable leak checking since the JVM triggers too many false positives.
+ export CFLAGS="$CFLAGS -fno-sanitize=leak"
+ export CXXFLAGS="$CXXFLAGS -fno-sanitize=leak"
fi
echo "---------------------------------------------------------------"
@@ -144,13 +151,24 @@ echo "CC=$CC"
echo "CXX=$CXX"
echo "CFLAGS=$CFLAGS"
echo "CXXFLAGS=$CXXFLAGS"
+echo "RUSTFLAGS=$RUSTFLAGS"
echo "---------------------------------------------------------------"
BUILD_CMD="bash -eux $SRC/build.sh"
+# Set +u temporarily to continue even if GOPATH and OSSFUZZ_RUSTPATH are undefined.
+set +u
# We need to preserve source code files for generating a code coverage report.
# We need exact files that were compiled, so copy both $SRC and $WORK dirs.
-COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH $OUT"
+COPY_SOURCES_CMD="cp -rL --parents $SRC $WORK /usr/include /usr/local/include $GOPATH $OSSFUZZ_RUSTPATH /rustc $OUT"
+set -u
+
+if [ "$FUZZING_LANGUAGE" = "rust" ]; then
+ # Copy rust std lib to its path with a hash.
+ export rustch=`rustc --version --verbose | grep commit-hash | cut -d' ' -f2`
+ mkdir -p /rustc/$rustch/
+ cp -r /rust/rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/ /rustc/$rustch/
+fi
if [ "${BUILD_UID-0}" -ne "0" ]; then
adduser -u $BUILD_UID --disabled-password --gecos '' builder
diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl
index dc6624459..d6509c74c 100644
--- a/infra/base-images/base-builder/compile_afl
+++ b/infra/base-images/base-builder/compile_afl
@@ -22,6 +22,8 @@
# AFL++ settings.
export AFL_LLVM_MODE_WORKAROUND=0
export AFL_ENABLE_DICTIONARY=0
+export AFL_ENABLE_CMPLOG=1
+export AFL_LAF_CHANCE=3
# Start compiling afl++.
echo "Copying precompiled afl++"
@@ -45,19 +47,32 @@ export ASAN_OPTIONS="detect_leaks=0:symbolize=0:detect_odr_violation=0:abort_on_
# AFL compile option roulette. It is OK if they all happen together.
-# 40% chance to perform CMPLOG
+# 20% chance for CTX-2 coverage instrumentation (Caller conTeXt sensitive
+# edge coverage).
+test $(($RANDOM % 100)) -lt 20 && {
+ export AFL_LLVM_INSTRUMENT=CLASSIC,CTX-2
+ export AFL_ENABLE_CMPLOG=0
+ export AFL_LAF_CHANCE=30
+}
+
+# 40% chance to create a dictionary.
+test $(($RANDOM % 100)) -lt 40 && {
+ export AFL_ENABLE_DICTIONARY=1
+}
+
+# 60% chance to perform CMPLOG/REDQUEEN.
rm -f "$OUT/afl_cmplog.txt"
-test $(($RANDOM % 10)) -lt 4 && {
+test "$AFL_ENABLE_CMPLOG" = "1" -a $(($RANDOM % 100)) -lt 60 && {
export AFL_LLVM_CMPLOG=1
touch "$OUT/afl_cmplog.txt"
}
-# 10% chance to perform LAF_INTEL
-test $(($RANDOM % 10)) -lt 1 && {
+# 3% chance to perform COMPCOV/LAF_INTEL.
+test $(($RANDOM % 100)) -lt $AFL_LAF_CHANCE && {
export AFL_LLVM_LAF_ALL=1
}
-# If the targets wants a dictionary - then create one.
+# Create a dictionary if one is wanted.
test "$AFL_ENABLE_DICTIONARY" = "1" && {
export AFL_LLVM_DICT2FILE="$OUT/afl++.dict"
}
diff --git a/infra/base-images/base-builder/compile_go_fuzzer b/infra/base-images/base-builder/compile_go_fuzzer
index 2342800fb..dd8c9f6a1 100755
--- a/infra/base-images/base-builder/compile_go_fuzzer
+++ b/infra/base-images/base-builder/compile_go_fuzzer
@@ -29,7 +29,7 @@ cd $GOPATH/src/$path || true
# in the case we are in the right directory, with go.mod but no go.sum
go mod tidy || true
# project was downloaded with go get if go list fails
-go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; }
+go list $tags $path || { cd $GOPATH/pkg/mod/ && cd `echo $path | cut -d/ -f1-3 | awk '{print $1"@*"}'`; } || cd -
# project does not have go.mod if go list fails again
go list $tags $path || { go mod init $path && go mod tidy ;}
@@ -42,7 +42,9 @@ if [[ $SANITIZER = *coverage* ]]; then
sed -i -e 's/mypackagebeingfuzzed/'$fuzzed_package'/' ./"${function,,}"_test.go
sed -i -e 's/TestFuzzCorpus/Test'$function'Corpus/' ./"${function,,}"_test.go
- fuzzed_repo=`echo $path | cut -d/ -f-3`
+ # The repo is the module path/name, which is already created above in case it doesn't exist,
+ # but not always the same as the module path. This is necessary to handle SIV properly.
+ fuzzed_repo=$(go list $tags -f {{.Module}} "$path")
abspath_repo=`go list -m $tags -f {{.Dir}} $fuzzed_repo || go list $tags -f {{.Dir}} $fuzzed_repo`
# give equivalence to absolute paths in another file, as go test -cover uses golangish pkg.Dir
echo "s=$fuzzed_repo"="$abspath_repo"= > $OUT/$fuzzer.gocovpath
diff --git a/infra/base-images/base-builder/install_go.sh b/infra/base-images/base-builder/install_go.sh
new file mode 100755
index 000000000..21138831c
--- /dev/null
+++ b/infra/base-images/base-builder/install_go.sh
@@ -0,0 +1,28 @@
+#!/bin/bash -eux
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+cd /tmp
+curl -O https://storage.googleapis.com/golang/getgo/installer_linux
+chmod +x ./installer_linux
+SHELL="bash" ./installer_linux
+rm -rf ./installer_linux
+
+echo 'Set "GOPATH=/root/go"'
+echo 'Set "PATH=$PATH:/root/.go/bin:$GOPATH/bin"'
+
+go get -u github.com/mdempsky/go114-fuzz-build
+ln -s $GOPATH/bin/go114-fuzz-build $GOPATH/bin/go-fuzz
diff --git a/infra/base-images/base-builder/install_java.sh b/infra/base-images/base-builder/install_java.sh
new file mode 100755
index 000000000..560c816a9
--- /dev/null
+++ b/infra/base-images/base-builder/install_java.sh
@@ -0,0 +1,37 @@
+#!/bin/bash -eux
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Install OpenJDK 15 and trim its size by removing unused components.
+cd /tmp
+curl -L -O https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db48ee2/7/GPL/openjdk-15.0.2_linux-x64_bin.tar.gz && \
+mkdir -p $JAVA_HOME
+tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \
+rm -f openjdk-15.0.2_linux-x64_bin.tar.gz
+rm -rf $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip
+
+# Install the latest Jazzer in $OUT.
+# jazzer_api_deploy.jar is required only at build-time, the agent and the
+# drivers are copied to $OUT as they need to be present on the runners.
+cd $SRC/
+git clone --depth=1 https://github.com/CodeIntelligenceTesting/jazzer && \
+cd jazzer
+bazel build --java_runtime_version=localjdk_15 -c opt --cxxopt="-stdlib=libc++" --linkopt=-lc++ \
+ //agent:jazzer_agent_deploy.jar //driver:jazzer_driver //driver:jazzer_driver_asan //driver:jazzer_driver_ubsan //agent:jazzer_api_deploy.jar
+cp bazel-bin/agent/jazzer_agent_deploy.jar bazel-bin/driver/jazzer_driver bazel-bin/driver/jazzer_driver_asan bazel-bin/driver/jazzer_driver_ubsan /usr/local/bin/
+cp bazel-bin/agent/jazzer_api_deploy.jar $JAZZER_API_PATH
+rm -rf ~/.cache/bazel ~/.cache/bazelisk
+rm -rf $SRC/jazzer
diff --git a/infra/base-images/base-builder/install_python.sh b/infra/base-images/base-builder/install_python.sh
new file mode 100755
index 000000000..b9c9a38c3
--- /dev/null
+++ b/infra/base-images/base-builder/install_python.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -eux
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+echo "ATHERIS INSTALL"
+unset CFLAGS CXXFLAGS
+pip3 install -v --no-cache-dir atheris>=2.0.6 pyinstaller==4.1
+rm -rf /tmp/*
diff --git a/infra/base-images/base-builder/install_rust.sh b/infra/base-images/base-builder/install_rust.sh
new file mode 100755
index 000000000..cbb461fd6
--- /dev/null
+++ b/infra/base-images/base-builder/install_rust.sh
@@ -0,0 +1,21 @@
+#!/bin/bash -eux
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+curl https://sh.rustup.rs | sh -s -- -y --default-toolchain=nightly --profile=minimal
+cargo install cargo-fuzz && rm -rf /rust/registry
+# Needed to recompile rust std library for MSAN
+rustup component add rust-src --toolchain nightly
diff --git a/infra/base-images/base-builder/install_swift.sh b/infra/base-images/base-builder/install_swift.sh
new file mode 100755
index 000000000..d88a7b5cd
--- /dev/null
+++ b/infra/base-images/base-builder/install_swift.sh
@@ -0,0 +1,66 @@
+#!/bin/bash -eux
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+
+SWIFT_PACKAGES="wget \
+ binutils \
+ git \
+ gnupg2 \
+ libc6-dev \
+ libcurl4 \
+ libedit2 \
+ libgcc-9-dev \
+ libpython2.7 \
+ libsqlite3-0 \
+ libstdc++-9-dev \
+ libxml2 \
+ libz3-dev \
+ pkg-config \
+ tzdata \
+ zlib1g-dev"
+SWIFT_SYMBOLIZER_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev"
+apt-get update && apt install -y $SWIFT_PACKAGES && \
+ apt install -y $SWIFT_SYMBOLIZER_PACKAGES --no-install-recommends
+
+
+wget https://swift.org/builds/swift-5.4.2-release/ubuntu2004/swift-5.4.2-RELEASE/swift-5.4.2-RELEASE-ubuntu20.04.tar.gz
+tar xzf swift-5.4.2-RELEASE-ubuntu20.04.tar.gz
+cp -r swift-5.4.2-RELEASE-ubuntu20.04/usr/* /usr/
+rm -rf swift-5.4.2-RELEASE-ubuntu20.04.tar.gz
+# TODO: Move to a seperate work dir
+git clone --depth 1 https://github.com/llvm/llvm-project.git
+cd llvm-project
+git apply ../llvmsymbol.diff --verbose
+cmake -G "Ninja" \
+ -DLIBCXX_ENABLE_SHARED=OFF \
+ -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
+ -DLIBCXXABI_ENABLE_SHARED=OFF \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DLLVM_TARGETS_TO_BUILD=X86 \
+ -DCMAKE_C_COMPILER=clang \
+ -DCMAKE_CXX_COMPILER=clang++ \
+ -DLLVM_BUILD_TESTS=OFF \
+ -DLLVM_INCLUDE_TESTS=OFF llvm
+ninja -j$(nproc) llvm-symbolizer
+cp bin/llvm-symbolizer /usr/local/bin/llvm-symbolizer-swift
+
+cd $SRC
+rm -rf llvm-project llvmsymbol.diff
+
+# TODO: Cleanup packages
+apt-get remove --purge -y wget zlib1g-dev
+apt-get autoremove -y
diff --git a/infra/base-images/base-builder/llvmsymbol.diff b/infra/base-images/base-builder/llvmsymbol.diff
new file mode 100644
index 000000000..70181bf39
--- /dev/null
+++ b/infra/base-images/base-builder/llvmsymbol.diff
@@ -0,0 +1,50 @@
+diff --git a/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt b/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt
+index acfb3bd0e..a499ee2e0 100644
+--- a/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt
++++ b/llvm/lib/DebugInfo/Symbolize/CMakeLists.txt
+@@ -12,4 +12,11 @@ add_llvm_component_library(LLVMSymbolize
+ Object
+ Support
+ Demangle
+- )
++
++ LINK_LIBS
++ /usr/lib/swift_static/linux/libswiftCore.a
++ /usr/lib/swift_static/linux/libicui18nswift.a
++ /usr/lib/swift_static/linux/libicuucswift.a
++ /usr/lib/swift_static/linux/libicudataswift.a
++ /usr/lib/x86_64-linux-gnu/libstdc++.so.6
++)
+diff --git a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+index fb4875f79..0030769ee 100644
+--- a/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
++++ b/llvm/lib/DebugInfo/Symbolize/Symbolize.cpp
+@@ -36,6 +36,13 @@
+ #include <cassert>
+ #include <cstring>
+
++
++extern "C" char *swift_demangle(const char *mangledName,
++ size_t mangledNameLength,
++ char *outputBuffer,
++ size_t *outputBufferSize,
++ uint32_t flags);
++
+ namespace llvm {
+ namespace symbolize {
+
+@@ -678,6 +685,14 @@ LLVMSymbolizer::DemangleName(const std::string &Name,
+ free(DemangledName);
+ return Result;
+ }
++ if (!Name.empty() && Name.front() == '$') {
++ char *DemangledName = swift_demangle(Name.c_str(), Name.length(), 0, 0, 0);
++ if (DemangledName) {
++ std::string Result = DemangledName;
++ free(DemangledName);
++ return Result;
++ }
++ }
+
+ if (DbiModuleDescriptor && DbiModuleDescriptor->isWin32Module())
+ return std::string(demanglePE32ExternCFunc(Name));
diff --git a/infra/base-images/base-builder/write_labels.py b/infra/base-images/base-builder/write_labels.py
index 6766e37fe..92a820a43 100755
--- a/infra/base-images/base-builder/write_labels.py
+++ b/infra/base-images/base-builder/write_labels.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");