[tensorflow] Integrate tensorflow with OSSfuzz (#1937)

* Integrate tensorflow with OSSfuzz

* Don't install already installed dependencies

* Add environment variable to disable logging.

Some fuzzers are outputting too much information due to logging and
that's not ok when fuzzing. Use TF_CPP_MIN_LOG_LEVEL envvar to disable
printing of the log messages.

* Remove the environment variable as it is not used at runtime and we have a workaround in the pipeline

* Workaround bazel bug (bazelbuild/bazel#6697)

* Don't compile with -O2, instead remove line causing bazel bug

* Downgrade Bazel to use the last version that TF supports

* Speed-up building by building fuzzers in parallel

* Add corpora and dictionaries

2 files changed, 175 insertions, 0 deletions

diff --git a/projects/tensorflow/Dockerfile b/projects/tensorflow/Dockerfile
new file mode 100644
index 000000000..e1395fc6f
--- /dev/null
+++ b/projects/tensorflow/Dockerfile
@@ -0,0 +1,40 @@
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+FROM gcr.io/oss-fuzz-base/base-builder
+MAINTAINER mihaimaruseac@google.com
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
+        python-dev \
+        rsync \
+        && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# Install Bazel
+RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list
+RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add -
+RUN apt-get update && apt-get install -y bazel
+
+# Downgrade Bazel to latest supported version (0.19.2)
+RUN curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/0.19.2/bazel-0.19.2-installer-linux-x86_64.sh
+RUN chmod +x ./bazel-0.19.2-installer-linux-x86_64.sh
+RUN ./bazel-0.19.2-installer-linux-x86_64.sh
+
+RUN git clone --depth 1 https://github.com/tensorflow/tensorflow tensorflow
+WORKDIR $SRC/tensorflow
+COPY build.sh $SRC/
diff --git a/projects/tensorflow/build.sh b/projects/tensorflow/build.sh
new file mode 100755
index 000000000..4d076d227
--- /dev/null
+++ b/projects/tensorflow/build.sh
@@ -0,0 +1,135 @@
+#!/bin/bash -eu
+# Copyright 2018 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+# Generate the list of fuzzers we have (only the base/op name).
+FUZZING_BUILD_FILE="tensorflow/core/kernels/fuzzing/BUILD"
+declare -r FUZZERS=$(
+  grep '^tf_ops_fuzz_target' ${FUZZING_BUILD_FILE} | cut -d'"' -f2
+)
+
+# Since Docker container has bazel-0.19 we need the following trick to allow
+# --config=monolithic and other needed flags.
+echo "  write_to_bazelrc('import %workspace%/tools/bazel.rc')" >> configure.py
+yes "" | ./configure
+
+# Add a few more flags to make sure fuzzers build and run successfully.
+# Note the c++11/libc++ flags to build using the same toolchain as the one used
+# to build libFuzzingEngine.
+CFLAGS="${CFLAGS} -fno-sanitize=vptr"
+CXXFLAGS="${CXXFLAGS} -fno-sanitize=vptr -std=c++11 -stdlib=libc++"
+
+# See https://github.com/bazelbuild/bazel/issues/6697
+sed '/::kM..SeedBytes/d' -i tensorflow/stream_executor/rng.cc
+
+# Due to statically linking boringssl dependency, we have to define one extra
+# flag when compiling for memory fuzzing (see the boringssl project).
+if [ "$SANITIZER" = "memory" ]
+then
+  CFLAGS="${CFLAGS} -DOPENSSL_NO_ASM=1"
+  CXXFLAGS="${CXXFLAGS} -DOPENSSL_NO_ASM=1"
+fi
+
+# All of the flags in $CFLAGS and $CXXFLAGS need to be passed to bazel too.
+# Also, pass in flags to ensure static build and to help in debugging failures.
+declare -r EXTRA_FLAGS="\
+--config=monolithic --dynamic_mode=off \
+--verbose_failures \
+$(
+for f in ${CFLAGS}; do
+  echo "--conlyopt=${f}" "--linkopt=${f}"
+done
+for f in ${CXXFLAGS}; do
+  echo "--cxxopt=${f}" "--linkopt=${f}"
+done
+)"
+
+# We need a new bazel function to build the actual binary.
+cat >> tensorflow/core/kernels/fuzzing/tf_ops_fuzz_target_lib.bzl << END
+
+def cc_tf(name):
+    native.cc_test(
+        name = name + "_fuzz",
+        deps = [
+            "//tensorflow/core/kernels/fuzzing:fuzz_session",
+            "//tensorflow/core/kernels/fuzzing:" + name + "_fuzz_lib",
+            "//tensorflow/cc:cc_ops",
+            "//tensorflow/cc:scope",
+            "//tensorflow/core:core_cpu",
+        ],
+    )
+END
+
+# Import this function in the proper BUILD file.
+cat >> ${FUZZING_BUILD_FILE} << END
+
+load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "cc_tf")
+
+END
+
+# And invoke it for all fuzzers.
+for fuzzer in ${FUZZERS}; do
+  echo cc_tf\(\"${fuzzer}\"\) >> ${FUZZING_BUILD_FILE}
+done
+
+# Since we force the environment, we expect bazel to fail during the linking of
+# each fuzzer. Hence, we will do the linking manually at the end of the process.
+# We just need to make sure we use the same invocation as bazel would use, so
+# use --verbose_failures (in ${EXTRA_FLAGS}) to get it and then encode it in the
+# following ${LINK_ARGS}.
+declare -r LINK_ARGS="\
+-pthread -fuse-ld=gold \
+-Wl,-no-as-needed -Wl,-z,relro,-z,now \
+-B/usr/local/bin -B/usr/bin -Wl,--gc-sections \
+"
+
+# This should always look as successful despite linking error mentioned above.
+bazel build ${EXTRA_FLAGS} -k //tensorflow/core/kernels/fuzzing:all || true
+
+# For each fuzzer target, we only have to link it manually to get the binary.
+for fuzzer in ${FUZZERS}; do
+  fz=${fuzzer}_fuzz
+
+  # Get the file with the parameters for linking or fail if it didn't exist.
+  lfile=`ls -1 bazel-bin/tensorflow/core/kernels/fuzzing/${fz}*.params | head -n1`
+
+  # Manually link everything.
+  ${CXX} ${CXXFLAGS} -lFuzzingEngine -o ${OUT}/${fz} ${LINK_ARGS} -Wl,@${lfile}
+done
+
+# For coverage, we need one extra step, see the envoy and grpc projects.
+if [ "$SANITIZER" = "coverage" ]
+then
+  declare -r REMAP_PATH=${OUT}/proc/self/cwd
+  mkdir -p ${REMAP_PATH}
+  rsync -aLk ${SRC}/tensorflow ${REMAP_PATH}
+fi
+
+# Now that all is done, we just have to copy the existing corpora and
+# dictionaries to have them available in the runtime environment.
+# The tweaks to the filenames below are to make sure corpora/dictionary have
+# similar names as the fuzzer binary.
+for dict in tensorflow/core/kernels/fuzzing/dictionaries/*; do
+  name=$(basename -- $dict)
+  cp ${dict} ${OUT}/${name/.dict/_fuzz.dict}
+done
+for corpus in tensorflow/core/kernels/fuzzing/corpus/*; do
+  name=$(basename -- $corpus)
+  zip ${OUT}/${name}_fuzz.zip ${corpus}/*
+done
+
+# Finally, make sure we don't accidentally run with stuff from the bazel cache.
+rm -f bazel-*