toolchain-utils: Add a tool to create llvm-extra ebuild.

llvm-extra ebuild is a slotted llvm compiler ebuild
that can be installed in parallel to the regualr llvm compiler.
Use of llvm-extra package is to provide an alternate clang in case
the normal clang compiler binary can't be used.

e.g. After a llvm upgrade, some package foo is miscompiled.
To avoid the miscompilation, we can use llvm-extra ebuild to install
the previous clang.

To use the clang installed by llvm-extra, modify the CFLAGS and
LDFLAGS to pass the patch of the clang binary installed by llvm-extra
package.
e.g. append-flags -Xclang-path=/usr/llvm-extra/version/clang
     append-ldflags -Xclang-path=/usr/llvm-extra/version/clang

This CL only adds the tool that can generate the llvm-extra
ebuilds from a given llvm ebuild.

BUG=chromium:773887
TEST= ./create_llvm_extra.sh /path/to/llvm/llvm-7.0_pre331547_p20180529-r8.ebuild
worked. A new llvm-extra ebuild was generated.

Change-Id: I7e3e20a2740f5e7135dbcd252cc271945c6ddd34
Reviewed-on: https://chromium-review.googlesource.com/1161213
Commit-Ready: Manoj Gupta <manojgupta@chromium.org>
Tested-by: Manoj Gupta <manojgupta@chromium.org>
Reviewed-by: Caroline Tice <cmtice@chromium.org>

2 files changed, 244 insertions, 0 deletions

diff --git a/llvm_extra/create_ebuild_file.py b/llvm_extra/create_ebuild_file.py
new file mode 100755
index 00000000..459e702a
--- /dev/null
+++ b/llvm_extra/create_ebuild_file.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python2
+
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import print_function
+
+import os
+import sys
+
+# This script takes an existing host llvm compiler ebuild and
+# creates another build that should be installable in a prefixed location.
+# The script patches a few lines in the llvm ebuild to make that happen.
+#
+# Since the script is based on the current llvm ebuild patterns,
+# it may need to be updated if those patterns change.
+#
+# This script should normally be invoked by the shell script
+# create_llvm_extra.sh .
+
+"""
+Below is an example of the expected diff of the newly generated ebuild with
+some explanation of the diffs.
+
+diff -Nuar llvm-pre7.0_pre335547_p20180529.ebuild newly-created-file.ebuild
+--- llvm-7.0_pre331547_p20180529-r8.ebuild
++++ newly-created-file.ebuild
+
+@@ -60,9 +60,9 @@ EGIT_REPO_URIS=(
+ fi
+
+ LICENSE="UoI-NCSA"
+-SLOT="0/${PV%%_*}"
++SLOT="${PV%%_p[[:digit:]]*}" # Creates a unique slot so that multiple copies
+                                of the new build can be installed.
+
+ KEYWORDS="-* amd64"
+
+ # Change USE flags to match llvm ebuild installtion. To see the set of flags
+ enabled in llvm compiler ebuild, run $ sudo emerge -pv llvm
+
+-IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi multitarget
++IUSE="debug +default-compiler-rt +default-libcxx doc libedit +libffi +multitarget
+        ncurses ocaml python llvm-next llvm-tot test xml video_cards_radeon"
+
+ COMMON_DEPEND="
+@@ -145,6 +145,7 @@ pkg_pretend() {
+ }
+
+ pkg_setup() {
+ # This Change is to install the files in $PREFIX.
++       export PREFIX="/usr/${PN}/${SLOT}"
+        pkg_pretend
+ }
+
+@@ -272,13 +273,13 @@
+        sed -e "/RUN/s/-warn-error A//" -i test/Bindings/OCaml/*ml  || die
+
+        # Allow custom cmake build types (like 'Gentoo')
+ # Convert use of PN to llvm in epatch commands.
+-       epatch "${FILESDIR}"/cmake/${PN}-3.8-allow_custom_cmake_build_types.patch
++       epatch "${FILESDIR}"/cmake/llvm-3.8-allow_custom_cmake_build_types.patch
+
+        # crbug/591436
+        epatch "${FILESDIR}"/clang-executable-detection.patch
+
+        # crbug/606391
+-       epatch "${FILESDIR}"/${PN}-3.8-invocation.patch
++       epatch "${FILESDIR}"/llvm-3.8-invocation.patch
+
+@@ -411,11 +412,14 @@ src_install() {
+                /usr/include/llvm/Config/llvm-config.h
+        )
+
++       MULTILIB_CHOST_TOOLS=() # No need to install any multilib tools/headers.
++       MULTILIB_WRAPPED_HEADERS=()
+        multilib-minimal_src_install
+ }
+
+ multilib_src_install() {
+        cmake-utils_src_install
++       return # No need to install any wrappers.
+
+        local wrapper_script=clang_host_wrapper
+        cat "${FILESDIR}/clang_host_wrapper.header" \
+@@ -434,6 +438,7 @@ multilib_src_install() {
+ }
+
+ multilib_src_install_all() {
++       return # No need to install common multilib files.
+        insinto /usr/share/vim/vimfiles
+        doins -r utils/vim/*/.
+        # some users may find it useful
+"""
+
+def process_line(line, text):
+  # Process the line and append to the text we want to generate.
+  # Check if line has any patterns that we want to handle.
+  newline = line.strip()
+  if newline.startswith('#'):
+    # Do not process comment lines.
+    text.append(line)
+  elif line.startswith('SLOT='):
+    # Change SLOT to "${PV%%_p[[:digit:]]*}"
+    SLOT_STRING='SLOT="${PV%%_p[[:digit:]]*}"\n'
+    text.append(SLOT_STRING)
+  elif line.startswith('IUSE') and 'multitarget' in line:
+    # Enable multitarget USE flag.
+    newline = line.replace('multitarget', '+multitarget')
+    text.append(newline)
+  elif line.startswith('pkg_setup()'):
+    # Setup PREFIX.
+    text.append(line)
+    text.append('\texport PREFIX="/usr/${PN}/${SLOT}"\n')
+  elif line.startswith('multilib_src_install_all()'):
+    text.append(line)
+    # Do not install any common files.
+    text.append('\treturn\n')
+  elif 'epatch ' in line:
+    # Convert any $PN or ${PN} in epatch files to llvm.
+    newline = line.replace('$PN', 'llvm')
+    newline = newline.replace('${PN}', 'llvm')
+    text.append(newline)
+  elif 'multilib-minimal_src_install' in line:
+    # Disable MULTILIB_CHOST_TOOLS and MULTILIB_WRAPPED_HEADERS
+    text.append('\tMULTILIB_CHOST_TOOLS=()\n')
+    text.append('\tMULTILIB_WRAPPED_HEADERS=()\n')
+    text.append(line)
+  elif 'cmake-utils_src_install' in line:
+    text.append(line)
+    # Do not install any wrappers.
+    text.append('\treturn\n')
+  else:
+    text.append(line)
+
+
+def main():
+  if len(sys.argv) != 3:
+     filename = os.path.basename(__file__)
+     print ('Usage: ', filename,' <input.ebuild> <output.ebuild>')
+     return 1
+
+  text = []
+  with open(sys.argv[1], 'r') as infile:
+    for line in infile:
+      process_line(line, text)
+
+  with open(sys.argv[2], 'w') as outfile:
+    outfile.write("".join(text))
+
+  return 0
+
+
+if __name__== "__main__":
+  sys.exit(main())
diff --git a/llvm_extra/create_llvm_extra.sh b/llvm_extra/create_llvm_extra.sh
new file mode 100755
index 00000000..6f34a0b2
--- /dev/null
+++ b/llvm_extra/create_llvm_extra.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script takes an existing llvm ebuild file and generate a llvm-extra
+# ebuild. The newly generated llvm-extra ebuild can be installed as a regular
+# host package.
+# The new ebuild should be generated in sys-devel/llvm-extra directory.
+# The script also copies all the files from files/ directory.
+# The generated llvm-extra ebuild is slotted so multiple instances of
+# llvm-extra ebuilds can be installed at same time.
+# The slot is derived based on the _pre<num> string in the llvm ebuild name.
+# e.g. For llvm-7.0_pre331547_p20180529-r8.ebuild, the slot will be
+# 7.0_pre331547.
+#
+# Usage:
+#  ./create_llvm_extra.sh /path/to/llvm-7.0_pre331547_p20180529-r8.ebuild
+#
+# To use the clang installed by llvm-extra, modify the CFLAGS and
+# LDFLAGS of a pckage to pass the patch of the clang binary installed by
+# the llvm-extra package.
+# e.g. append-flags -Xclang-path=/usr/llvm-extra/version/clang
+#      append-ldflags -Xclang-path=/usr/llvm-extra/version/clang
+#
+
+SCRIPT_DIR=$(realpath $(dirname "$0"))
+
+function check_cmd() {
+	if [[ "$#" -ne 1 ]]; then
+		echo "Exactly 1 argument expected"
+		echo "Usage $0 <path_to_llvm_ebuild>"
+		exit 1
+	fi
+	if [[ ! -f "$1" ]]; then
+		echo "$1 is not a file"
+		exit 1;
+	fi
+}
+
+function create_llvm_extra_ebuild() {
+	EBUILD_PREFIX=llvm-extra
+	EBUILD_DIR=$(dirname "$1")
+	EBUILD_FILE_NAME=$(basename "$1")
+	NEW_EBUILD_FILE_NAME="${EBUILD_FILE_NAME/llvm/$EBUILD_PREFIX}"
+	NEW_EBUILD_FILENAME_NO_EXT="${NEW_EBUILD_FILE_NAME%.*}"
+	NEW_EBUILD_DIR="${EBUILD_DIR}/../${EBUILD_PREFIX}"
+	NEW_EBUILD_PV="${NEW_EBUILD_FILENAME_NO_EXT#"$EBUILD_PREFIX-"}"
+	NEW_EBUILD_SLOT="${NEW_EBUILD_PV%%_p[[:digit:]]*}"
+
+	mkdir -p "${NEW_EBUILD_DIR}"
+	if [[ -d "${EBUILD_DIR}/files" ]]; then
+		cp -rf "${EBUILD_DIR}/files" "${NEW_EBUILD_DIR}"
+	fi
+
+	if [[ -f "${NEW_EBUILD_DIR}/${NEW_EBUILD_FILE_NAME}" ]]; then
+		echo "Removing existing ebuild file ${NEW_EBUILD_FILE_NAME}"
+		rm -f "${NEW_EBUILD_DIR}/${NEW_EBUILD_FILE_NAME}"
+	fi
+	# Generate the llvm-extra ebuild file.
+	"${SCRIPT_DIR}"/create_ebuild_file.py "$1" "${NEW_EBUILD_DIR}/${NEW_EBUILD_FILE_NAME}"
+	if [[ $? -ne 0 ]]; then
+		echo "Creation of ${NEW_EBUILD_DIR}/${NEW_EBUILD_FILE_NAME} failed"
+		exit 1
+	fi
+	echo "***"
+	echo "***"
+	echo "${NEW_EBUILD_DIR}/${NEW_EBUILD_FILE_NAME} has been created."
+
+	echo "***"
+	echo "Test if it builds by running \$ sudo emerge ${EBUILD_PREFIX}:${NEW_EBUILD_SLOT}"
+	echo "***"
+	echo "If it works, Go ahead and submit the newly generated ebuild"\
+	     "and any other files in ${NEW_EBUILD_DIR}."
+	echo "***"
+	echo "Don't forget to add sys-devel/${EBUILD_PREFIX}:${NEW_EBUILD_SLOT} to"\
+	     "the dependencies in virtual/target-chromium-os-sdk ebuild."
+	echo "***"
+	echo "***"
+}
+
+
+set -e
+# Sanity checks.
+check_cmd "${@}"
+# Create llvm-extra ebuild.
+create_llvm_extra_ebuild "${@}"