Merge changes from topic 'split-python'

* changes:
  Add build.py and fix up build-python.sh.
  Move build-python.sh to toolchain/python.

2 files changed, 570 insertions, 0 deletions

diff --git a/build-python.sh b/build-python.sh
new file mode 100755
index 0000000..135b2e1
--- /dev/null
+++ b/build-python.sh
@@ -0,0 +1,476 @@
+#!/bin/sh
+#
+# Copyright (C) 2012 The Android Open Source Project
+# Copyright (C) 2012 Ray Donnelly <mingw.android at gmail.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Rebuild the host Python binaries from sources.
+# Also copies any gnu libstdc++ pretty-printers
+# found in $TOOLCHAIN_SRC_DIR/gcc/gcc-*
+# and Joachim Reichel's stlport pretty printers
+# found in sources/host-tools/gdb-pretty-printers/stlport
+#
+
+# include common function and variable definitions
+. "$NDK_BUILDTOOLS_PATH/prebuilt-common.sh"
+. "$NDK_BUILDTOOLS_PATH/common-build-host-funcs.sh"
+
+PROGRAM_PARAMETERS=""
+PROGRAM_DESCRIPTION="\
+This program is used to rebuild one or more Python client programs from
+sources.
+
+By default, the script rebuilds Python for you host system [$HOST_TAG],
+but you can use --systems=<tag1>,<tag2>,.. to ask binaries that can run on
+several distinct systems. Each <tag> value in the list can be one of the
+following:
+
+   linux-x86
+   linux-x86_64
+   windows
+   windows-x86  (equivalent to 'windows')
+   windows-x86_64
+   darwin-x86
+   darwin-x86_64
+
+For example, here's how to rebuild Python 2.7.5 on Linux
+for six different systems:
+
+  $PROGNAME --build-dir=/path/to/toolchain/src \n \
+    --python-version=2.7.5 \n \
+    --systems=linux-x86,linux-x86_64,windows,windows-x86_64,darwin-x86,darwin-x86_64"
+
+TOOLCHAIN_SRC_DIR=
+register_var_option "--toolchain-src-dir=<path>" TOOLCHAIN_SRC_DIR "Select toolchain source directory"
+
+PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
+register_var_option "--python-version=<version>" PYTHON_VERSION "Select Python version."
+
+NDK_DIR=$ANDROID_NDK_ROOT
+register_var_option "--ndk-dir=<path>" NDK_DIR "Select NDK install directory."
+
+PACKAGE_DIR=
+register_var_option "--package-dir=<path>" PACKAGE_DIR "Package prebuilt tarballs into directory."
+
+BUILD_DIR=
+register_var_option "--build-dir=<path>" BUILD_DIR "Build Python into directory"
+
+bh_register_options
+register_try64_option
+register_canadian_option
+register_jobs_option
+
+extract_parameters "$@"
+
+if [ -n "$PARAMETERS" ]; then
+    panic "This script doesn't take parameters, only options. See --help"
+fi
+
+if [ -z "$TOOLCHAIN_SRC_DIR" ]; then
+    panic "Please use --toolchain-src-dir=<path> to select toolchain source directory."
+fi
+check_toolchain_src_dir "$TOOLCHAIN_SRC_DIR"
+TOOLCHAIN_SRC_DIR=`cd $TOOLCHAIN_SRC_DIR; pwd`
+
+BH_HOST_SYSTEMS=$(commas_to_spaces $BH_HOST_SYSTEMS)
+AUTO_BUILD="no"
+
+if [ "$MINGW" = "yes" ]; then
+    BH_HOST_SYSTEMS="windows"
+    log "Auto-config: --systems=windows"
+fi
+
+if [ "$DARWIN" = "yes" ]; then
+    BH_HOST_SYSTEMS="darwin-x86"
+    log "Auto-config: --systems=darwin-x86"
+fi
+
+determine_systems ()
+{
+    local IN_SYSTEMS="$1"
+    local OUT_SYSTEMS
+
+    for SYSTEM in $IN_SYSTEMS; do
+        if [ "$TRY64" = "yes" ]; then
+            case $SYSTEM in
+                darwin-x86|linux-x86|windows-x86)
+                    SYSTEM=${SYSTEM%%x86}x86_64
+                    ;;
+                windows)
+                    SYSTEM=windows-x86_64
+                    ;;
+            esac
+        else
+            # 'windows-x86' causes substitution
+            # failure at the packing stage.
+            case $SYSTEM in
+                windows-x86)
+                    SYSTEM=windows
+                    ;;
+            esac
+        fi
+        OUT_SYSTEMS="$OUT_SYSTEMS $SYSTEM"
+    done
+    echo $OUT_SYSTEMS
+}
+
+BH_HOST_SYSTEMS=$(determine_systems "$BH_HOST_SYSTEMS")
+
+# Build python for build machine automatically
+if [ "$(bh_list_contains $BH_BUILD_TAG $BH_HOST_SYSTEMS)" = "no" ]; then
+    BH_HOST_SYSTEMS="$BH_BUILD_TAG $BH_HOST_SYSTEMS"
+    AUTO_BUILD="yes"
+fi
+
+# Python needs to execute itself during its build process, so must build the build
+# Python first. It should also be an error if not asked to build for build machine.
+BH_HOST_SYSTEMS=$(bh_sort_systems_build_first "$BH_HOST_SYSTEMS")
+
+download_package ()
+{
+    # Assume the packages are already downloaded under $ARCHIVE_DIR
+    local PKG_URL=$1
+    local PKG_NAME=$(basename $PKG_URL)
+
+    case $PKG_NAME in
+        *.tar.bz2)
+            PKG_BASENAME=${PKG_NAME%%.tar.bz2}
+            ;;
+        *.tar.gz)
+            PKG_BASENAME=${PKG_NAME%%.tar.gz}
+            ;;
+        *)
+            panic "Unknown archive type: $PKG_NAME"
+    esac
+
+    if [ ! -f "$ARCHIVE_DIR/$PKG_NAME" ]; then
+        log "Downloading $PKG_URL..."
+        (cd $ARCHIVE_DIR && run curl -L -o "$PKG_NAME" "$PKG_URL")
+        fail_panic "Can't download '$PKG_URL'"
+    fi
+
+    if [ ! -d "$SRC_DIR/$PKG_BASENAME" ]; then
+        log "Uncompressing $PKG_URL into $SRC_DIR"
+        case $PKG_NAME in
+            *.tar.bz2)
+                run tar xjf $ARCHIVE_DIR/$PKG_NAME -C $SRC_DIR
+                ;;
+            *.tar.gz)
+                run tar xzf $ARCHIVE_DIR/$PKG_NAME -C $SRC_DIR
+                ;;
+            *)
+                panic "Unknown archive type: $PKG_NAME"
+                ;;
+        esac
+        fail_panic "Can't uncompress $ARCHIVE_DIR/$PKG_NAME"
+    fi
+}
+
+if [ -z "$BUILD_DIR" ] ; then
+    BUILD_DIR=$TMPDIR/buildhost
+fi
+
+bh_setup_build_dir $BUILD_DIR
+
+if [ "$BH_BUILD_MODE" = "debug" ] ; then
+   PYDEBUG="-with-pydebug"
+fi
+
+# Sanity check that we have the right compilers for all hosts
+for SYSTEM in $BH_HOST_SYSTEMS; do
+    bh_setup_build_for_host $SYSTEM
+done
+
+TEMP_DIR=$BUILD_DIR/tmp
+# Download and unpack source packages from official sites
+ARCHIVE_DIR=$TEMP_DIR/archive
+STAMP_DIR=$TEMP_DIR/timestamps
+BUILD_DIR=$TEMP_DIR/build-$HOST_TAG
+
+mkdir -p $BUILD_DIR
+
+PROGDIR=`dirname $0`
+PROGDIR=$(cd $PROGDIR && pwd)
+
+# Sanity check for all Python versions.
+for VERSION in $(commas_to_spaces $PYTHON_VERSION); do
+    PYTHON_SRCDIR=$TOOLCHAIN_SRC_DIR/python/Python-$VERSION
+    if [ ! -d "$PYTHON_SRCDIR" ]; then
+        panic "Missing source directory: $PYTHON_SRCDIR"
+    fi
+done
+
+arch_to_qemu_arch ()
+{
+    case $1 in
+        x86)
+            echo i386
+            ;;
+        *)
+            echo $1
+            ;;
+    esac
+}
+
+# $1: host system tag
+# $2: python version
+build_host_python ()
+{
+    local SRCDIR=$TOOLCHAIN_SRC_DIR/python/Python-$2
+    local BUILDDIR=$BH_BUILD_DIR/build-python-$1-$2
+    local INSTALLDIR=$(python_build_install_dir $1 $2)
+    local ARGS TEXT
+
+    if [ ! -f "$SRCDIR/configure" ]; then
+        panic "Missing configure script in $SRCDIR"
+    fi
+
+    # Currently, 2.7.5 and 3.3.0 builds generate $SRCDIR/Lib/_sysconfigdata.py, unless it
+    # already exists (in which case it ends up wrong anyway!)... this should really be in
+    # the build directory instead.
+    if [ ! -f "$SRCDIR/Lib/_sysconfigdata.py" ]; then
+        log "Removing old $SRCDIR/Lib/_sysconfigdata.py"
+        rm -f $SRCDIR/Lib/_sysconfigdata.py
+    fi
+
+    ARGS=" --prefix=$INSTALLDIR"
+
+    ARGS=$ARGS" --build=$BH_BUILD_CONFIG"
+    ARGS=$ARGS" --host=$BH_HOST_CONFIG"
+    ARGS=$ARGS" --with-build-sysroot"
+    ARGS=$ARGS" $PYDEBUG"
+    ARGS=$ARGS" --disable-ipv6"
+
+    mkdir -p "$BUILDDIR" && rm -rf "$BUILDDIR"/*
+    cd "$BUILDDIR" &&
+    bh_setup_host_env
+
+    CFG_SITE=
+    # Need to add -L$HOST_STATIC_LIBDIR to LDSHARED if need
+    # any static host libs.
+    export LDSHARED="$CC -shared "
+    if [ ! $BH_HOST_TAG = $BH_BUILD_TAG ]; then
+
+        # Cross compiling.
+        CFG_SITE=$BUILDDIR/config.site
+
+        # Ideally would remove all of these configury hacks by
+        # patching the issues.
+
+        if [ $1 = darwin-x86 -o $1 = darwin-x86_64 ]; then
+            echo "ac_cv_file__dev_ptmx=no"              > $CFG_SITE
+            echo "ac_cv_file__dev_ptc=no"              >> $CFG_SITE
+            echo "ac_cv_have_long_long_format=yes"     >> $CFG_SITE
+            if [ $1 = darwin-x86 ] ; then
+                echo "ac_osx_32bit=yes"                >> $CFG_SITE
+            elif [ $1 = darwin-x86_64 ] ; then
+                echo "ac_osx_32bit=no"                 >> $CFG_SITE
+            fi
+            echo "ac_cv_have_sendfile=no"              >> $CFG_SITE
+            # I could change AC_MSG_CHECKING(LDSHARED) in configure.ac
+            # to check $host instead of $ac_sys_system/$ac_sys_release
+            # but it handles loads of platforms
+            # and I can only test on three, so instead...
+            export LDSHARED="$CC -bundle -undefined dynamic_lookup"
+        elif [ $1 = windows-x86 -o $1 = windows-x86_64 ]; then
+            echo "ac_cv_file__dev_ptmx=no"              > $CFG_SITE
+            echo "ac_cv_file__dev_ptc=no"              >> $CFG_SITE
+            CFLAGS=$CFLAGS" -D__USE_MINGW_ANSI_STDIO=1"
+            CXXFLAGS=$CXXFLAGS" -D__USE_MINGW_ANSI_STDIO=1"
+        elif [ $1 = linux-x86 -o $1 = linux-x86_64 ]; then
+            echo "ac_cv_file__dev_ptmx=yes"             > $CFG_SITE
+            echo "ac_cv_file__dev_ptc=no"              >> $CFG_SITE
+            echo "ac_cv_have_long_long_format=yes"     >> $CFG_SITE
+            echo "ac_cv_pthread_system_supported=yes"  >> $CFG_SITE
+            echo "ac_cv_working_tzset=yes"             >> $CFG_SITE
+            echo "ac_cv_little_endian_double=yes"      >> $CFG_SITE
+        fi
+
+        if [ "$BH_HOST_OS" = "$BH_BUILD_OS" ]; then
+            # Only cross compiling from arch perspective.
+            # qemu causes failures as cross-compilation is not detected
+            # if a test executable can be run successfully, so we test
+            # for qemu-${BH_HOST_ARCH} and qemu-${BH_HOST_ARCH}-static
+            # and panic if either are found.
+            QEMU_HOST_ARCH=$(arch_to_qemu_arch $BH_HOST_ARCH)
+            if [ ! -z "$(which qemu-$QEMU_HOST_ARCH 2>/dev/null)" -o \
+                 ! -z "$(which qemu-$QEMU_HOST_ARCH-static 2>/dev/null)" ] ; then
+               dump "Installed qemu(s) ($(which qemu-$QEMU_HOST_ARCH 2>/dev/null) $(which qemu-$QEMU_HOST_ARCH-static 2>/dev/null))" \
+                      "will prevent this build from working."
+            fi
+        fi
+    else
+        if [ $1 = darwin-x86 -o $1 = darwin-x86_64 ]; then
+            export LDSHARED="$CC -bundle -undefined dynamic_lookup"
+        fi
+    fi
+
+    TEXT="$(bh_host_text) python-$BH_HOST_CONFIG-$2:"
+
+    touch $SRCDIR/Include/graminit.h
+    touch $SRCDIR/Python/graminit.c
+    echo "" > $SRCDIR/Parser/pgen.stamp
+    touch $SRCDIR/Parser/Python.asdl
+    touch $SRCDIR/Parser/asdl.py
+    touch $SRCDIR/Parser/asdl_c.py
+    touch $SRCDIR/Include/Python-ast.h
+    touch $SRCDIR/Python/Python-ast.c
+
+    # By default, the Python build will force the following compiler flags
+    # after our own CFLAGS:
+    #   -g -fwrap -O3 -Wall -Wstrict-prototypes
+    #
+    # The '-g' is unfortunate because it makes the generated binaries
+    # much larger than necessary, and stripping them after the fact is
+    # a bit delicate when cross-compiling. To avoid this, define a
+    # custom OPT variable here (see Python-2.7.5/configure.ac) when
+    # generating non stripped builds.
+    if [ "$BH_BUILD_MODE" = "release" ]; then
+      OPT="-fwrapv -O3 -Wall -Wstrict-prototypes"
+      export OPT
+    fi
+
+    dump "$TEXT Building"
+    export CONFIG_SITE=$CFG_SITE &&
+    run "$SRCDIR"/configure $ARGS
+    fail_panic "Could not configure Python!"
+
+    #
+    # Note 1:
+    # sharedmods is a phony target, but it's a dependency of both "make all" and also
+    # "make install", this causes it to fail on Windows as it tries to rename pydoc3
+    # to pydoc3.3 twice, and the second time aroud the file exists. So instead, we
+    # just do make install.
+    #
+    # Note 2:
+    # Can't run make install with -j as from the Makefile:
+    # install:	 altinstall bininstall maninstall
+    #  meaning altinstall and bininstall are kicked off at the same time
+    #  but actually, bininstall depends on altinstall being run first
+    #  due to libainstall: doing
+    #  $(INSTALL_SCRIPT) python-config $(DESTDIR)$(BINDIR)/python$(VERSION)-config
+    #  and bininstall: doing
+    #  (cd $(DESTDIR)$(BINDIR); $(LN) -s python$(VERSION)-config python2-config)
+    #  Though the real fix is to make bininstall depend on libainstall.
+    run make -j$NUM_JOBS
+    fail_panic "Could not build Python!"
+
+    run make install
+    fail_panic "Could not install Python!"
+
+    # Pretty printers.
+    PYPPDIR="$INSTALLDIR/share/pretty-printers/"
+
+    # .. for gnu stdlibc++
+    GCC_DIRS=$(find $TOOLCHAIN_SRC_DIR/gcc/ -maxdepth 1 -name "gcc-*" -type d)
+    for GCC_DIR in $GCC_DIRS; do
+        (
+        if [ -d "$GCC_DIR/libstdc++-v3/python" ]; then
+            cd "$GCC_DIR/libstdc++-v3/python"
+            [ -d "$PYPPDIR/libstdcxx/$(basename $GCC_DIR)" ] || mkdir -p "$PYPPDIR/libstdcxx/$(basename $GCC_DIR)"
+            run find . -path "*.py" -exec cp {} "$PYPPDIR/libstdcxx/$(basename $GCC_DIR)/" \;
+        fi
+        )
+    done
+
+    # .. for STLPort
+    run cp -rf $NDK_DIR/sources/host-tools/gdb-pretty-printers/stlport/gppfs-0.2 $PYPPDIR/stlport
+}
+
+need_build_host_python ()
+{
+    bh_do build_host_python $1 $2
+}
+
+# Install host Python binaries and support files to the NDK install dir.
+# $1: host tag
+# $2: python version
+install_host_python ()
+{
+    local SRCDIR="$(python_build_install_dir $1 $2)"
+    local DSTDIR="$NDK_DIR/$(python_ndk_install_dir $1 $2)"
+
+    need_build_host_python $1 $2
+
+    if [ $AUTO_BUILD != "yes" -o $1 != $BH_BUILD_TAG ]; then
+        dump "$(bh_host_text) python-$BH_HOST_ARCH-$2: Installing"
+        run copy_directory "$SRCDIR/bin"     "$DSTDIR/bin"
+        run copy_directory "$SRCDIR/lib"     "$DSTDIR/lib"
+        run copy_directory "$SRCDIR/share"   "$DSTDIR/share"
+        run copy_directory "$SRCDIR/include" "$DSTDIR/include"
+        # remove unneeded files
+        run rm -rf "$DSTDIR/share/man"
+        run rm -rf "$DSTDIR/share/pretty-printers/libstdcxx/gcc-4.9.*"
+        run rm -rf "$DSTDIR/share/pretty-printers/libstdcxx/gcc-4.9-*"
+        run rm -rf "$DSTDIR/share/pretty-printers/libstdcxx/gcc-[lm]*"
+    fi
+}
+
+# $1: host tag
+# $2: python version
+need_install_host_python ()
+{
+    local SRCDIR="$(python_build_install_dir $1 $2)"
+
+    bh_do install_host_python $1 $2
+
+    # make sharedmods (setup.py) needs to use the build machine's Python
+    # for the other hosts to build correctly.
+    if [ $BH_BUILD_TAG = $BH_HOST_TAG ]; then
+        export PATH=$SRCDIR/bin:$PATH
+    fi
+}
+
+# Package host Python binaries into a tarball
+# $1: host tag
+# $2: python version
+package_host_python ()
+{
+    local BLDDIR="$(python_build_install_dir $1 $2)"
+    local SRCDIR="$(python_ndk_install_dir $1 $2)"
+    # This is similar to BLDDIR=${BLDDIR%%$SRCDIR} (and requires we use windows and not windows-x86)
+    BLDDIR=$(echo "$BLDDIR" | sed "s/$(echo "$SRCDIR" | sed -e 's/\\/\\\\/g' -e 's/\//\\\//g' -e 's/&/\\\&/g')//g")
+    local PACKAGENAME=ndk-python-$(install_dir_from_host_tag $1).tar.bz2
+    local PACKAGE="$PACKAGE_DIR/$PACKAGENAME"
+
+    dump "$(bh_host_text) $PACKAGENAME: Packaging"
+    run pack_archive "$PACKAGE" "$BLDDIR" "$SRCDIR"
+}
+
+need_package_host_python ()
+{
+    bh_do package_host_python $1 $2
+}
+
+PYTHON_VERSION=$(commas_to_spaces $PYTHON_VERSION)
+ARCHS=$(commas_to_spaces $ARCHS)
+
+# Let's build this
+for SYSTEM in $BH_HOST_SYSTEMS; do
+    bh_setup_build_for_host $SYSTEM
+    for VERSION in $PYTHON_VERSION; do
+        need_install_host_python $SYSTEM $VERSION
+    done
+done
+
+if [ "$PACKAGE_DIR" ]; then
+    for SYSTEM in $BH_HOST_SYSTEMS; do
+        bh_setup_build_for_host $SYSTEM
+        if [ $AUTO_BUILD != "yes" -o $SYSTEM != $BH_BUILD_TAG ]; then
+            for VERSION in $PYTHON_VERSION; do
+                need_package_host_python $SYSTEM $VERSION
+            done
+        fi
+    done
+fi
diff --git a/build.py b/build.py
new file mode 100755
index 0000000..d7cb088
--- /dev/null
+++ b/build.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2015 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Builds Python for the NDK."""
+from __future__ import print_function
+
+import argparse
+import inspect
+import os
+import subprocess
+import sys
+
+
+def android_path(path=''):
+    top = os.getenv('ANDROID_BUILD_TOP', '')
+    return os.path.realpath(os.path.join(top, path))
+
+
+def get_default_package_dir():
+    return android_path('out/ndk')
+
+
+def get_default_host():
+    if sys.platform in ('linux', 'linux2'):
+        return 'linux'
+    elif sys.platform == 'darwin':
+        return 'darwin'
+    else:
+        raise RuntimeError('Unsupported host: {}'.format(sys.platform))
+
+
+class ArgParser(argparse.ArgumentParser):
+    def __init__(self):
+        super(ArgParser, self).__init__(
+            description=inspect.getdoc(sys.modules[__name__]),
+            formatter_class=argparse.RawDescriptionHelpFormatter)
+
+        self.add_argument(
+            '--host', choices=('darwin', 'linux', 'windows', 'windows64'),
+            default=get_default_host(),
+            help='Build binaries for given OS (e.g. linux).')
+        self.add_argument(
+            '--package-dir', help='Directory to place the packaged GCC.',
+            default=get_default_package_dir())
+
+
+def main():
+    args = ArgParser().parse_args()
+
+    if 'ANDROID_BUILD_TOP' not in os.environ:
+        top = os.path.join(os.path.dirname(__file__), '../..')
+        os.environ['ANDROID_BUILD_TOP'] = os.path.realpath(top)
+
+    os.chdir(android_path('toolchain/python'))
+
+    toolchain_path = android_path('toolchain')
+    ndk_path = android_path('ndk')
+
+    ndk_build_tools_path = android_path('ndk/build/tools')
+    build_env = dict(os.environ)
+    build_env['NDK_BUILDTOOLS_PATH'] = ndk_build_tools_path
+    build_env['ANDROID_NDK_ROOT'] = ndk_path
+
+    toolchain_dir_arg = '--toolchain-src-dir={}'.format(toolchain_path)
+    package_dir_arg = '--package-dir={}'.format(args.package_dir)
+
+    build_cmd = [
+        'bash', 'build-python.sh', toolchain_dir_arg, package_dir_arg,
+        '--verbose',
+    ]
+
+    if args.host in ('windows', 'windows64'):
+        build_cmd.append('--mingw')
+
+    if args.host != 'windows':
+        build_cmd.append('--try-64')
+
+    subprocess.check_call(build_cmd, env=build_env)
+
+if __name__ == '__main__':
+    main()