From db02d9252e8ec27b2131cb2db70ea635d056eb79 Mon Sep 17 00:00:00 2001
From: jonathanmetzman <31354670+jonathanmetzman@users.noreply.github.com>
Date: Wed, 3 Mar 2021 19:10:35 -0800
Subject: [AFL++][base-builder] Precompile afl++ (#5290)

Precompile AFL like we already do for honggfuzz.
This saves about a minute in compilation time of AFL targets by doing it in base-builder
It only adds about 30 MB to the image size.
---
 infra/base-images/base-builder/Dockerfile          |  7 ++---
 infra/base-images/base-builder/compile_afl         | 32 ++++----------------
 infra/base-images/base-builder/compile_honggfuzz   |  4 +--
 infra/base-images/base-builder/precompile_afl      | 35 ++++++++++++++++++++++
 .../base-images/base-builder/precompile_honggfuzz  |  7 ++---
 5 files changed, 48 insertions(+), 37 deletions(-)
 create mode 100755 infra/base-images/base-builder/precompile_afl

(limited to 'infra/base-images')

diff --git a/infra/base-images/base-builder/Dockerfile b/infra/base-images/base-builder/Dockerfile
index b726812e5..057904707 100644
--- a/infra/base-images/base-builder/Dockerfile
+++ b/infra/base-images/base-builder/Dockerfile
@@ -171,9 +171,6 @@ ENV LIB_FUZZING_ENGINE="/usr/lib/libFuzzingEngine.a"
 # TODO: remove after tpm2 catchup.
 ENV FUZZER_LDFLAGS ""
 
-ENV PRECOMPILED_DIR="/usr/lib/precompiled"
-RUN mkdir $PRECOMPILED_DIR
-
 WORKDIR $SRC
 
 # TODO: switch to -b stable once we can.
@@ -189,11 +186,13 @@ RUN cd $SRC && \
     rm -rf examples $SRC/oss-fuzz.tar.gz
 
 COPY compile compile_afl compile_dataflow compile_libfuzzer compile_honggfuzz \
-    compile_go_fuzzer precompile_honggfuzz srcmap write_labels.py /usr/local/bin/
+    compile_go_fuzzer precompile_honggfuzz precompile_afl srcmap \
+    write_labels.py /usr/local/bin/
 
 COPY detect_repo.py /opt/cifuzz/
 COPY ossfuzz_coverage_runner.go $GOPATH
 
 RUN precompile_honggfuzz
+RUN precompile_afl
 
 CMD ["compile"]
diff --git a/infra/base-images/base-builder/compile_afl b/infra/base-images/base-builder/compile_afl
index 17762d38e..eef0b0193 100644
--- a/infra/base-images/base-builder/compile_afl
+++ b/infra/base-images/base-builder/compile_afl
@@ -19,37 +19,12 @@
 # The 'env|grep' setup ensures we do not trigger the linter.
 # The variables need to be set to "1" here - or before running this script.
 
-# If enabled this provides a safe work around if afl-clang-fast ever break:
-env | grep -qw AFL_LLVM_MODE_WORKAROUND || {
-  # needed until llvm 13 works:
-  AFL_LLVM_MODE_WORKAROUND=0
-}
-
-# If a dictionary should be generated based on comparisons at compile time:
-env | grep -qw AFL_ENABLE_DICTIONARY || {
-  AFL_ENABLE_DICTIONARY=1
-}
-
 # Start compiling afl++.
-echo "Compiling afl++"
+echo "Copying precompiled afl++"
 
-# Build and copy afl++ tools necessary for fuzzing.
+# Copy afl++ tools necessary for fuzzing.
 pushd $SRC/aflplusplus > /dev/null
 
-# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it
-# down with sanitizers.
-SAVE_CXXFLAGS=$CXXFLAGS
-SAVE_CFLAGS=$CFLAGS
-unset CXXFLAGS
-unset CFLAGS
-export AFL_IGNORE_UNKNOWN_ENVS=1
-make clean
-AFL_NO_X86=1 PYTHON_INCLUDE=/ make
-CFLAGS=$SAVE_CFLAGS
-CXXFLAGS=$SAVE_CXXFLAGS
-
-# Build afl++ driver with existing CFLAGS, CXXFLAGS.
-make -C utils/aflpp_driver
 cp -f libAFLDriver.a $LIB_FUZZING_ENGINE
 
 # Some important projects include libraries, copy those even when they don't
@@ -81,6 +56,9 @@ test $(($RANDOM % 10)) -lt 1 && {
   export AFL_LLVM_LAF_ALL=1
 }
 
+export AFL_LLVM_MODE_WORKAROUND=0
+export AFL_ENABLE_DICTIONARY=0
+
 # In case afl-clang-fast ever breaks, this is a workaround:
 test "$AFL_LLVM_MODE_WORKAROUND" = "1" && {
   export CC=clang
diff --git a/infra/base-images/base-builder/compile_honggfuzz b/infra/base-images/base-builder/compile_honggfuzz
index 362a0a598..f86e8426d 100755
--- a/infra/base-images/base-builder/compile_honggfuzz
+++ b/infra/base-images/base-builder/compile_honggfuzz
@@ -17,8 +17,8 @@
 
 echo "Skipping compilation; using precompiled honggfuzz"
 
-cp $PRECOMPILED_DIR/honggfuzz.a $LIB_FUZZING_ENGINE
-cp $PRECOMPILED_DIR/honggfuzz $OUT/
+cp $SRC/honggfuzz/honggfuzz.a $LIB_FUZZING_ENGINE
+cp $SRC/honggfuzz/honggfuzz $OUT/
 
 # Custom coverage flags, roughly in sync with:
 # https://github.com/google/honggfuzz/blob/oss-fuzz/hfuzz_cc/hfuzz-cc.c
diff --git a/infra/base-images/base-builder/precompile_afl b/infra/base-images/base-builder/precompile_afl
new file mode 100755
index 000000000..d6e71f2c3
--- /dev/null
+++ b/infra/base-images/base-builder/precompile_afl
@@ -0,0 +1,35 @@
+#!/bin/bash -eu
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+echo "Precompiling AFLplusplus"
+
+pushd $SRC/aflplusplus > /dev/null
+make clean
+# Unset CFLAGS and CXXFLAGS while building AFL since we don't want to slow it
+# down with sanitizers.
+SAVE_CXXFLAGS=$CXXFLAGS
+SAVE_CFLAGS=$CFLAGS
+unset CXXFLAGS
+unset CFLAGS
+export AFL_IGNORE_UNKNOWN_ENVS=1
+make clean
+AFL_NO_X86=1 PYTHON_INCLUDE=/ make
+make -C utils/aflpp_driver
+
+popd > /dev/null
+
+echo "Done."
diff --git a/infra/base-images/base-builder/precompile_honggfuzz b/infra/base-images/base-builder/precompile_honggfuzz
index 2565bb83f..df6bb2b75 100755
--- a/infra/base-images/base-builder/precompile_honggfuzz
+++ b/infra/base-images/base-builder/precompile_honggfuzz
@@ -15,7 +15,7 @@
 #
 ################################################################################
 
-echo -n "Precompiling honggfuzz to $PRECOMPILED_DIR..."
+echo "Precompiling honggfuzz"
 export BUILD_OSSFUZZ_STATIC=true
 
 PACKAGES=(
@@ -37,10 +37,9 @@ CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD" make
 
 # libhfuzz.a will be added by CC/CXX linker directly during linking,
 # but it's defined here to satisfy the build infrastructure
-ar rcs $PRECOMPILED_DIR/honggfuzz.a libhfuzz/*.o libhfcommon/*.o
-cp honggfuzz $PRECOMPILED_DIR/
+ar rcs honggfuzz.a libhfuzz/*.o libhfcommon/*.o
 popd > /dev/null
 
 apt-get remove -y --purge ${PACKAGES[@]}
 apt-get autoremove -y
-echo " done."
+echo "Done."
-- 
cgit v1.2.3