add integrated cross-compile & qemu support

This uses the existing CrOS toolchains & boards to cross-compile for arm
and mipsel boards.  It also includes qemu support for running tests.

BUG=b/26741388
TEST=`make cross-arm all-libs` downloads & builds w/arm

Change-Id: Ie974b1c3c2b61759e87f87bab535068a1c31a727
Reviewed-on: https://weave-review.googlesource.com/3000
Reviewed-by: Alex Vakulenko <avakulenko@google.com>

6 files changed, 257 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
index 86c649b..c102afc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 *~
 /out/
+/third_party/cross
 /third_party/include
 /third_party/lib
 /third_party/libevhtp
diff --git a/Makefile b/Makefile
index 00f237c..8cbb0e7 100644
--- a/Makefile
+++ b/Makefile
@@ -77,7 +77,7 @@ DEFS_TEST := \
 out/$(BUILD_MODE)/libweave.so : out/$(BUILD_MODE)/libweave_common.a
 	$(CXX) -shared -Wl,-soname=libweave.so -o $@ -Wl,--whole-archive $^ -Wl,--no-whole-archive -lcrypto -lexpat -lpthread -lrt
 
-include file_lists.mk third_party/third_party.mk examples/examples.mk tests.mk
+include cross.mk file_lists.mk third_party/third_party.mk examples/examples.mk tests.mk
 
 ###
 # src/
diff --git a/README.md b/README.md
index c640581..26f1cf7 100644
--- a/README.md
+++ b/README.md
@@ -120,6 +120,24 @@ make all-examples
 
 See [the examples README](/examples/daemon/README.md) for details.
 
+### Cross-compiling
+
+The build supports transparently downloading & using a few cross-compilers.
+Just add `cross-<arch>` to the command line in addition to the target you
+want to actually build.
+
+This will cross-compile for an armv7 (hard float) target:
+
+```
+make cross-arm all-libs
+```
+
+This will cross-compile for a mips (little endian) target:
+
+```
+make cross-mipsel all-libs
+```
+
 # Testing
 
 ### Run tests
@@ -135,6 +153,16 @@ or
 make testall
 ```
 
+### Cross-testing
+
+The build supports using qemu to run non-native tests.
+
+This will run armv7 tests through qemu:
+
+```
+make cross-arm testall
+```
+
 # Making changes
 
 The [Android Developing site](https://source.android.com/source/developing.html)
diff --git a/cross.mk b/cross.mk
new file mode 100644
index 0000000..e833ca9
--- /dev/null
+++ b/cross.mk
@@ -0,0 +1,40 @@
+# Copyright 2016 The Weave Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Logic to easily run cross-compiling tests.
+
+CROSS_ROOT := $(PWD)/third_party/cross
+CROSS_FLAGS :=
+DOWNLOAD_CROSS_TOOLCHAINS := no
+QEMU_BASE := $(CROSS_ROOT)/app-emulation/qemu/usr/bin
+
+define cross-setup-gcc
+.PHONY: $(1)
+ifneq (,$$(findstring $(1),$$(MAKECMDGOALS)))
+DOWNLOAD_CROSS_TOOLCHAINS := yes
+CHOST := $(2)
+BOARD := $(3)
+CROSS := $$(CROSS_ROOT)/$$(CHOST)/bin/$$(CHOST)-
+CC := $$(CROSS)gcc
+CXX := $$(CROSS)g++
+AR := $$(CROSS)ar
+CROSS_FLAGS += $(5)
+QEMU := $$(QEMU_BASE)/$(4) -L $$(CROSS_ROOT)/$$(BOARD)
+endif
+endef
+
+# Whitespace matters with arguments, so we can't make this more readable :/.
+$(eval $(call cross-setup-gcc,cross-arm,armv7a-cros-linux-gnueabi,arm-generic-full,qemu-arm,-mhard-float))
+$(eval $(call cross-setup-gcc,cross-mipsel,mipsel-cros-linux-gnu,mipsel-o32-generic-full,qemu-mipsel))
+$(eval $(call cross-setup-gcc,cross-x86,i686-pc-linux-gnu,x86-generic-full,qemu-i386))
+$(eval $(call cross-setup-gcc,cross-x86_64,x86_64-cros-linux-gnu,amd64-generic-full,qemu-x86_64))
+
+ifeq ($(DOWNLOAD_CROSS_TOOLCHAINS),yes)
+ifeq (,$(wildcard third_party/cross/$(BOARD)))
+CROSS_FETCH_OUT := $(shell ./third_party/get_cross.sh >&2)
+endif
+CROSS_FLAGS += --sysroot $(CROSS_ROOT)/$(BOARD)
+CC += $(CROSS_FLAGS)
+CXX += $(CROSS_FLAGS)
+endif
diff --git a/tests.mk b/tests.mk
index d0042db..cd711ef 100644
--- a/tests.mk
+++ b/tests.mk
@@ -12,6 +12,7 @@ TEST_ENV ?=
 ifeq (1, $(CLANG))
   TEST_ENV += ASAN_SYMBOLIZER_PATH=$(shell which llvm-symbolizer-3.6)
 endif
+TEST_ENV += $(QEMU)
 
 weave_test_obj_files := $(WEAVE_TEST_SRC_FILES:%.cc=out/$(BUILD_MODE)/%.o)
 
diff --git a/third_party/get_cross.sh b/third_party/get_cross.sh
new file mode 100755
index 0000000..f69291c
--- /dev/null
+++ b/third_party/get_cross.sh
@@ -0,0 +1,186 @@
+#!/bin/bash
+# Copyright 2016 The Weave Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+set -e
+
+SCRIPT=$(readlink -f "$0")
+THIRD_PARTY=$(dirname "${SCRIPT}")
+cd "${THIRD_PARTY}"
+
+OUT="cross"
+DISTDIR="${OUT}/distfiles"
+
+CROS_OVERLAY_URL="https://chromium.googlesource.com/chromiumos/overlays/chromiumos-overlay/+/master/chromeos"
+CONF_SDK_LATEST="${CROS_OVERLAY_URL}/binhost/host/sdk_version.conf"
+
+SDK_BUCKET="https://commondatastorage.googleapis.com/chromiumos-sdk"
+BINPKG_BUCKET="https://commondatastorage.googleapis.com/chromeos-prebuilt"
+CROS_BUCKET="https://commondatastorage.googleapis.com/chromeos-image-archive"
+
+PKGS=(
+  app-emulation/qemu
+)
+TARGETS=(
+#  aarch64-cros-linux-gnu
+  armv7a-cros-linux-gnueabi
+  i686-pc-linux-gnu
+#  mips-cros-linux-gnu
+  mipsel-cros-linux-gnu
+  x86_64-cros-linux-gnu
+)
+BOARDS=(
+#  aarch64-generic-full
+  amd64-generic-full
+  arm-generic-full
+#  mips-o32-generic-full
+  mipsel-o32-generic-full
+  x86-generic-full
+)
+
+usage() {
+  cat <<EOF
+Usage: get_cross.sh
+
+Download cross-compilers for building & testing against other arches.
+EOF
+  exit 0
+}
+
+get_gitiles() {
+  local url="$1" data
+  data=$(curl -s "${url}?format=TEXT")
+  echo "${data}" | base64 -d
+}
+
+json() {
+  local file="$1" arg="$2"
+  python <<EOF
+import json
+print(json.load(open("${file}"))${arg})
+EOF
+}
+
+fetch() {
+  local url=$1
+  file="${2:-${DISTDIR}/${url##*/}}"
+  if [[ ! -e ${file} ]]; then
+    printf '[downloading] '
+    mkdir -p "${DISTDIR}"
+    wget "${url}" -O "${file}"
+  fi
+}
+
+unpack() {
+  local out="$1" file="$2"
+  printf '[unpacking] '
+  rm -rf "${out}"
+  mkdir -p "${out}"
+  tar xf "${file}" -C "${out}"
+}
+
+fetch_pkgs() {
+  local pkg
+  local sub_url url file manifest
+  local out ver_file old_ver ver
+
+  # Grab a few helper packages.
+  printf 'Getting SDK manifest ... '
+  sub_url="cros-sdk-${SDK_LATEST_VERSION}.tar.xz.Manifest"
+  url="${SDK_BUCKET}/${sub_url}"
+  fetch "${url}"
+  manifest=${file}
+  printf '%s\n' "${manifest}"
+
+  for pkg in "${PKGS[@]}"; do
+    printf 'Getting binpkg %s ... ' "${pkg}"
+    ver=$(json "${manifest}" '["packages"]["app-emulation/qemu"][0][0]')
+    sub_url="host/amd64/amd64-host/chroot-${SDK_LATEST_VERSION}/packages/${pkg}-${ver}.tbz2"
+    url="${BINPKG_BUCKET}/${sub_url}"
+    fetch "${url}"
+
+    out="${OUT}/${pkg}"
+    ver_file="${out}/.ver"
+    old_ver=$(cat "${ver_file}" 2>/dev/null || :)
+    if [[ "${old_ver}" != "${ver}" ]]; then
+      unpack "${out}" "${file}"
+      echo "${ver}" > "${ver_file}"
+    fi
+
+    printf '%s\n' "${ver}"
+  done
+}
+
+fetch_toolchains() {
+  local target
+  local sub_url url file
+  local out ver_file ver
+
+  # Download the base toolchains.
+  for target in "${TARGETS[@]}"; do
+    printf 'Getting toolchain for %s ... ' "${target}"
+
+    sub_url="${TC_PATH/\%(target)s/${target}}"
+    url="${SDK_BUCKET}/${sub_url}"
+    file="${DISTDIR}/${url##*/}"
+    fetch "${url}"
+
+    out="${OUT}/${target}"
+    ver_file="${out}/.ver"
+    ver=$(cat "${ver_file}" 2>/dev/null || :)
+    if [[ "${ver}" != "${SDK_LATEST_VERSION}" ]]; then
+      unpack "${out}" "${file}"
+      echo "${SDK_LATEST_VERSION}" > "${ver_file}"
+    fi
+
+    printf '%s\n' "${sub_url}"
+  done
+}
+
+fetch_sysroots() {
+  local board
+  local board_latest_url sub_url url file
+  local out ver_file ver
+
+  # Get the full sysroot.
+  for board in "${BOARDS[@]}"; do
+    printf 'Getting sysroot for %s ... ' "${board}"
+    board_latest_url="${CROS_BUCKET}/${board}/LATEST-master"
+    if ! board_ver=$(curl --fail -s "${board_latest_url}"); then
+      echo 'error: not found'
+      continue
+    fi
+
+    url="${CROS_BUCKET}/${board}/${board_ver}/sysroot_chromeos-base_chromeos-chrome.tar.xz"
+    file="${DISTDIR}/${board}-${board_ver}-${url##*/}"
+    fetch "${url}" "${file}"
+
+    out="${OUT}/${board}"
+    ver_file="${out}/.ver"
+    ver=$(cat "${ver_file}" 2>/dev/null || :)
+    if [[ "${ver}" != "${board_ver}" ]]; then
+      unpack "${out}" "${file}"
+      echo "${board_ver}" > "${ver_file}"
+    fi
+
+    printf '%s\n' "${board_ver}"
+  done
+}
+
+main() {
+  if [[ $# -ne 0 ]]; then
+    usage
+  fi
+
+  # Get the current SDK versions.
+  printf 'Getting CrOS SDK version ... '
+  data=$(get_gitiles "${CONF_SDK_LATEST}")
+  eval "${data}"
+  echo "${SDK_LATEST_VERSION}"
+
+  fetch_pkgs
+  fetch_toolchains
+  fetch_sysroots
+}
+main "$@"