Android updates for Python 3.10 am: 2b32bc35db

Original change: https://android-review.googlesource.com/c/platform/external/python/cpython3/+/2037671

Change-Id: I08982956e2902ca3a275992942572ae395da1496

5 files changed, 35 insertions, 51 deletions

diff --git a/Android.bp b/Android.bp
index 8c10965fe6..26b6fe6f6f 100644
--- a/Android.bp
+++ b/Android.bp
@@ -139,16 +139,16 @@ cpython3_cc_defaults {
             local_include_dirs: ["android/bionic/pyconfig"],
         },
         android_arm: {
-            cflags: ["-DSOABI=\"cpython-39android-arm-android-bionic\""],
+            cflags: ["-DSOABI=\"cpython-310android-arm-android-bionic\""],
         },
         android_arm64: {
-            cflags: ["-DSOABI=\"cpython-39android-arm64-android-bionic\""],
+            cflags: ["-DSOABI=\"cpython-310android-arm64-android-bionic\""],
         },
         android_x86: {
-            cflags: ["-DSOABI=\"cpython-39android-x86-android-bionic\""],
+            cflags: ["-DSOABI=\"cpython-310android-x86-android-bionic\""],
         },
         android_x86_64: {
-            cflags: ["-DSOABI=\"cpython-39android-x86_64-android-bionic\""],
+            cflags: ["-DSOABI=\"cpython-310android-x86_64-android-bionic\""],
         },
         host: {
             // Host builds are moving to prebuilts, which has a higher risk of
@@ -161,14 +161,14 @@ cpython3_cc_defaults {
             cflags: [
                 "-Wno-deprecated-declarations",
                 "-Wno-pointer-arith",
-                "-DSOABI=\"cpython-39android-x86_64-darwin\"",
+                "-DSOABI=\"cpython-310android-x86_64-darwin\"",
             ],
         },
         linux_bionic: {
             // NB linux_bionic is a 'host' architecture but it uses the bionic libc like 'android'
             // targets so use the android pyconfig.
             local_include_dirs: ["android/bionic/pyconfig"],
-            cflags: ["-DSOABI=\"cpython-39android-x86_64-linux-bionic\""],
+            cflags: ["-DSOABI=\"cpython-310android-x86_64-linux-bionic\""],
         },
         linux_glibc_x86: {
             enabled: false,
@@ -178,11 +178,11 @@ cpython3_cc_defaults {
         },
         linux_glibc_x86_64: {
             local_include_dirs: ["android/linux_x86_64/pyconfig"],
-            cflags: ["-DSOABI=\"cpython-39android-x86_64-linux-gnu\""],
+            cflags: ["-DSOABI=\"cpython-310android-x86_64-linux-gnu\""],
         },
         linux_musl_x86_64: {
             local_include_dirs: ["android/linux_x86_64/pyconfig"],
-            cflags: ["-DSOABI=\"cpython-39android-x86_64-linux-gnu\""],
+            cflags: ["-DSOABI=\"cpython-310android-x86_64-linux-gnu\""],
         },
         windows: {
             enabled: false,
@@ -211,19 +211,13 @@ cc_library_static {
     cflags: ["-DPy_BUILD_CORE", "-DPLATLIBDIR=\"lib\""],
     srcs: [
         // Makefile.pre.in PEGEN_OBJS
-        "Parser/pegen/pegen.c",
-        "Parser/pegen/parse.c",
-        "Parser/pegen/parse_string.c",
-        "Parser/pegen/peg_api.c",
+        "Parser/pegen.c",
+        "Parser/parser.c",
+        "Parser/string_parser.c",
+        "Parser/peg_api.c",
 
         // Makefile.pre.in PARSER_OBJS
-        "Parser/acceler.c",
-        "Parser/grammar1.c",
-        "Parser/listnode.c",
         "Parser/myreadline.c",
-        "Parser/node.c",
-        "Parser/parser.c",
-        "Parser/parsetok.c",
         "Parser/token.c",
         "Parser/tokenizer.c",
 
@@ -270,6 +264,7 @@ cc_library_static {
         "Objects/typeobject.c",
         "Objects/unicodeobject.c",
         "Objects/unicodectype.c",
+        "Objects/unionobject.c",
         "Objects/weakrefobject.c",
 
         // Makefile.pre.in PYTHON_OBJS
@@ -294,7 +289,6 @@ cc_library_static {
         "Python/getcopyright.c",
         "Python/getplatform.c",
         "Python/getversion.c",
-        "Python/graminit.c",
         "Python/hamt.c",
         "Python/hashtable.c",
         "Python/import.c",
@@ -305,7 +299,6 @@ cc_library_static {
         "Python/mysnprintf.c",
         "Python/mystrtoul.c",
         "Python/pathconfig.c",
-        "Python/peephole.c",
         "Python/preconfig.c",
         "Python/pyarena.c",
         "Python/pyctype.c",
@@ -329,6 +322,7 @@ cc_library_static {
         "Python/dtoa.c",
         "Python/formatter_unicode.c",
         "Python/fileutils.c",
+        "Python/suggestions.c",
         "Python/dynload_shlib.c",
     ],
 
@@ -352,7 +346,7 @@ cc_defaults {
     name: "py3-launcher-defaults",
     defaults: ["py3-interp-defaults", "cpython3-interp-host-prebuilts"],
     cflags: [
-        "-DVERSION=\"3.9\"",
+        "-DVERSION=\"3.10\"",
         "-DVPATH=\"\"",
         "-DPREFIX=\"\"",
         "-DEXEC_PREFIX=\"\"",
@@ -535,6 +529,7 @@ cc_library_static {
             ],
         },
         darwin: {
+            cflags: ["-D__APPLE_USE_RFC_3542"],
             srcs: [":py3-c-modules-darwin_x86_64"],
             static_libs: [
                 "libopenssl_ssl",
diff --git a/android/Setup.local b/android/Setup.local
index f374c50fd2..0113ebb225 100644
--- a/android/Setup.local
+++ b/android/Setup.local
@@ -11,7 +11,6 @@ _random _randommodule.c
 _bisect _bisectmodule.c
 _heapq _heapqmodule.c
 _pickle _pickle.c
-atexit atexitmodule.c
 _json _json.c
 _lsprof _lsprof.c rotatingtree.c
 unicodedata unicodedata.c
@@ -24,7 +23,6 @@ fcntl fcntlmodule.c
 pwd pwdmodule.c
 grp grpmodule.c
 select selectmodule.c
-parser parsermodule.c
 mmap mmapmodule.c
 syslog syslogmodule.c
 _xxsubinterpreters _xxsubinterpretersmodule.c
diff --git a/android/regen.sh b/android/regen.sh
index 1c0e235c33..ea040767a0 100755
--- a/android/regen.sh
+++ b/android/regen.sh
@@ -24,8 +24,10 @@ DIR=`uname | tr 'A-Z' 'a-z'`_x86_64
 mkdir -p $DIR/pyconfig
 cd $DIR
 
+export CLANG_VERSION=$(cd $ANDROID_BUILD_TOP; build/soong/scripts/get_clang_version.py)
+
 if [ $DIR == "linux_x86_64" ]; then
-  export CC="$ANDROID_BUILD_TOP/prebuilts/clang/host/linux-x86/clang-r399163b/bin/clang"
+  export CC="$ANDROID_BUILD_TOP/prebuilts/clang/host/linux-x86/$CLANG_VERSION/bin/clang"
   export CFLAGS="--sysroot=$ANDROID_BUILD_TOP/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8/sysroot"
   export LDFLAGS="--sysroot=$ANDROID_BUILD_TOP/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8/sysroot -B$ANDROID_BUILD_TOP/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8/lib/gcc/x86_64-linux/4.8.3 -L$ANDROID_BUILD_TOP/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8/lib/gcc/x86_64-linux/4.8.3 -L$ANDROID_BUILD_TOP/prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.17-4.8/x86_64-linux/lib64"
 fi
@@ -39,20 +41,11 @@ cd tmp
 ../../../configure
 
 if [ $DIR == "darwin_x86_64" ]; then
-  # getentropy is not safe on <10.12, which we still target
-  sed -ibak "s%#define HAVE_GETENTROPY 1%/* #undef HAVE_GETENTROPY */%" pyconfig.h
-  # utimensat and futimens are not safe on <10.13, which we still target
-  sed -ibak "s%#define HAVE_UTIMENSAT 1%/* #undef HAVE_UTIMENSAT */%" pyconfig.h
-  sed -ibak "s%#define HAVE_FUTIMENS 1%/* #undef HAVE_FUTIMENS */%" pyconfig.h
   # preadv and pwritev are not safe on <11, which we still target
   sed -ibak "s%#define HAVE_PREADV 1%/* #undef HAVE_PREADV */%" pyconfig.h
   sed -ibak "s%#define HAVE_PWRITEV 1%/* #undef HAVE_PWRITEV */%" pyconfig.h
-  # _dyld_shared_cache_contains_path is not safe on <11, which we still target
-  sed -ibak "s%#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH 1%/* #undef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH */%" pyconfig.h
 fi
 
-cp pyconfig.h ../pyconfig/
-
 if [ $DIR == "linux_x86_64" ]; then
   mkdir -p ../../bionic/pyconfig
   cp pyconfig.h ../../bionic/pyconfig/
@@ -75,8 +68,13 @@ if [ $DIR == "linux_x86_64" ]; then
   sed -i 's%#define SIZEOF_TIME_T .*%#define SIZEOF_TIME_T SIZEOF_LONG%' $bionic_pyconfig
   sed -i 's%#define SIZEOF_UINTPTR_T .*%#define SIZEOF_UINTPTR_T SIZEOF_LONG%' $bionic_pyconfig
   sed -i 's%#define SIZEOF_VOID_P .*%#define SIZEOF_VOID_P SIZEOF_LONG%' $bionic_pyconfig
+
+  # Changes to support musl
+  sed -i "s%#define HAVE_DECL_RTLD_DEEPBIND 1%#ifdef __GLIBC__\n#define HAVE_DECL_RTLD_DEEPBIND 1\n#endif%" pyconfig.h
 fi
 
+cp pyconfig.h ../pyconfig/
+
 function generate_srcs() {
   #
   # Generate config.c
diff --git a/kokoro/build.py b/kokoro/build.py
index 1f061241ff..cde1a8d7a2 100755
--- a/kokoro/build.py
+++ b/kokoro/build.py
@@ -53,7 +53,7 @@ def build_autoconf_target(host, python_src, build_dir, install_dir,
             config_cmd.append('--enable-universalsdk')
         config_cmd.append('--with-universal-archs=universal2')
 
-        MAC_MIN_VERSION = '10.9'
+        MAC_MIN_VERSION = '10.14'
         cflags.append('-mmacosx-version-min={}'.format(MAC_MIN_VERSION))
         cflags.append('-DMACOSX_DEPLOYMENT_TARGET={}'.format(MAC_MIN_VERSION))
         cflags.extend(['-arch', 'arm64'])
@@ -64,17 +64,10 @@ def build_autoconf_target(host, python_src, build_dir, install_dir,
         # Disable functions to support old macOS. See https://bugs.python.org/issue31359
         # Fails the build if any new API is used.
         cflags.append('-Werror=unguarded-availability')
-        # Disables unavailable functions.
-        disable_funcs = [
-            # New in 10.13
-            'utimensat', 'futimens',
-            # New in 10.12
-            'getentropy', 'clock_getres', 'clock_gettime', 'clock_settime',
-            # New in 10.10
-            'fstatat', 'faccessat', 'fchmodat', 'fchownat', 'linkat', 'fdopendir',
-            'mkdirat', 'renameat', 'unlinkat', 'readlinkat', 'symlinkat', 'openat',
-        ]
-        config_cmd.extend('ac_cv_func_{}=no'.format(f) for f in disable_funcs)
+        # We're building with a macOS 11+ SDK, so this should be set, but
+        # configure doesn't find it because of the unguarded-availability error
+        # combined with and older -mmacosx-version-min
+        cflags.append('-DHAVE_DYLD_SHARED_CACHE_CONTAINS_PATH=1')
     elif host == Host.Linux:
         # Quoting for -Wl,-rpath,$ORIGIN:
         #  - To link some binaries, make passes -Wl,-rpath,\$ORIGIN to shell.
@@ -98,7 +91,7 @@ def build_autoconf_target(host, python_src, build_dir, install_dir,
         # Linker will embed this path to all binaries linking this library.
         # Since configure does not give us a chance to set -install_name, we have
         # to edit the library afterwards.
-        libpython = 'libpython3.9.dylib'
+        libpython = 'libpython3.10.dylib'
         subprocess.check_call(['make',
                                '-j{}'.format(multiprocessing.cpu_count()),
                                libpython],
diff --git a/kokoro/prepare_windows_externals.py b/kokoro/prepare_windows_externals.py
index 0065de1764..75efe68d53 100644
--- a/kokoro/prepare_windows_externals.py
+++ b/kokoro/prepare_windows_externals.py
@@ -68,18 +68,18 @@ def set_text_element(root: minidom.Element, tag: str, new_text: str) -> None:
     node.data = new_text
 
 
-def patch_python_vcxproj_for_licenses():
+def patch_python_for_licenses():
     # Python already handles bzip2 and libffi itself.
     notice_files = [
         TOP / 'external/zlib/LICENSE',
         TOP / 'toolchain/xz/COPYING',
     ]
 
-    xml_path = PYTHON_SRC / 'PCbuild/python.vcxproj'
+    xml_path = PYTHON_SRC / 'PCbuild/regen.targets'
     proj = read_xml_file(xml_path)
 
-    # Pick the unconditional <LicenseFiles> element and add extra notices to the end.
-    elements = proj.getElementsByTagName('LicenseFiles')
+    # Pick the unconditional <_LicenseSources> element and add extra notices to the end.
+    elements = proj.getElementsByTagName('_LicenseSources')
     (element,) = [e for e in elements if not e.hasAttribute('Condition')]
     includes = element.getAttribute('Include').split(';')
     for notice in notice_files:
@@ -177,7 +177,7 @@ def main() -> None:
     shutil.copy2(xz / 'windows/vs2017/config.h',
                  xz / 'windows/config.h')
 
-    patch_python_vcxproj_for_licenses()
+    patch_python_for_licenses()
     remove_modules_from_pcbuild_proj()
     build_using_cmake(TOP / 'out/libffi', TOP / 'external/libffi')
     build_using_cmake(TOP / 'out/zlib', TOP / 'external/zlib')