Move Android build and prefix directories from cross-build to Android

Author: mhsmith

Android/.gitignore

@@ -0,0 +1,3 @@
+/build
+/dist
+/prefix

Android/README.md

@@ -60,8 +60,8 @@ To do all steps in a single command, run:
 ./android.py build HOST
 ```
 
-In the end you should have a build Python in `cross-build/build`, and an Android
-build in `cross-build/HOST`.
+In the end you should have an Android build of Python and its supporting
+libraries in `prefix/HOST`.
 
 You can use `--` as a separator for any of the `configure`-related commands –
 including `build` itself – to pass arguments to the underlying `configure`
@@ -80,7 +80,7 @@ The test suite can be run on Linux, macOS, or Windows:
 * On Linux, the emulator needs access to the KVM virtualization interface, and
   a DISPLAY environment variable pointing at an X server.
 * On Windows, you won't be able to do the build on the same machine, so you'll
-  have to copy the `cross-build/HOST` directory from somewhere else.
+  have to copy the `prefix/HOST` directory from somewhere else.
 
 The test suite can usually be run on a device with 2 GB of RAM, but this is
 borderline, so you may need to increase it to 4 GB. As of Android

Android/android.py

@@ -22,9 +22,12 @@
 SCRIPT_NAME = Path(__file__).name
 CHECKOUT = Path(__file__).resolve().parent.parent
 ANDROID_DIR = CHECKOUT / "Android"
+BUILD_DIR = ANDROID_DIR / "build"
+DIST_DIR = ANDROID_DIR / "dist"
+PREFIX_DIR = ANDROID_DIR / "prefix"
 TESTBED_DIR = ANDROID_DIR / "testbed"
-CROSS_BUILD_DIR = CHECKOUT / "cross-build"
 
+HOSTS = ["aarch64-linux-android", "x86_64-linux-android"]
 APP_ID = "org.python.testbed"
 DECODE_ARGS = ("UTF-8", "backslashreplace")
 
@@ -58,12 +61,10 @@ def delete_glob(pattern):
             path.unlink()
 
 
-def subdir(name, *, clean=None):
-    path = CROSS_BUILD_DIR / name
-    if clean:
-        delete_glob(path)
+def subdir(parent, host, *, create=False):
+    path = parent / host
     if not path.exists():
-        if clean is None:
+        if not create:
             sys.exit(
                 f"{path} does not exist. Create it by running the appropriate "
                 f"`configure` subcommand of {SCRIPT_NAME}.")
@@ -83,7 +84,7 @@ def run(command, *, host=None, env=None, log=True, **kwargs):
         env_output = subprocess.run(
             f"set -eu; "
             f"HOST={host}; "
-            f"PREFIX={subdir(host)}/prefix; "
+            f"PREFIX={subdir(PREFIX_DIR, host)}; "
             f". {env_script}; "
             f"export",
             check=True, shell=True, text=True, stdout=subprocess.PIPE
@@ -111,19 +112,21 @@ def run(command, *, host=None, env=None, log=True, **kwargs):
 
 def build_python_path():
     """The path to the build Python binary."""
-    build_dir = subdir("build")
-    binary = build_dir / "python"
+    build_subdir = subdir(BUILD_DIR, "build")
+    binary = build_subdir / "python"
     if not binary.is_file():
         binary = binary.with_suffix(".exe")
         if not binary.is_file():
             raise FileNotFoundError("Unable to find `python(.exe)` in "
-                                    f"{build_dir}")
+                                    f"{build_subdir}")
 
     return binary
 
 
 def configure_build_python(context):
-    os.chdir(subdir("build", clean=context.clean))
+    if context.clean:
+        clean("build")
+    os.chdir(subdir(BUILD_DIR, "build", create=True))
 
     command = [relpath(CHECKOUT / "configure")]
     if context.args:
@@ -132,7 +135,7 @@ def configure_build_python(context):
 
 
 def make_build_python(context):
-    os.chdir(subdir("build"))
+    os.chdir(subdir(BUILD_DIR, "build"))
     run(["make", "-j", str(os.cpu_count())])
 
 
@@ -153,17 +156,16 @@ def download(url, target_dir="."):
 
 
 def configure_host_python(context):
-    host_dir = subdir(context.host, clean=context.clean)
+    if context.clean:
+        clean(context.host)
 
-    prefix_dir = host_dir / "prefix"
-    if not prefix_dir.exists():
-        prefix_dir.mkdir()
-        os.chdir(prefix_dir)
+    prefix_subdir = subdir(PREFIX_DIR, context.host, create=True)
+    if not (prefix_subdir / "include").exists():
+        os.chdir(prefix_subdir)
         unpack_deps(context.host)
 
-    build_dir = host_dir / "build"
-    build_dir.mkdir(exist_ok=True)
-    os.chdir(build_dir)
+    build_subdir = subdir(BUILD_DIR, context.host, create=True)
+    os.chdir(build_subdir)
 
     command = [
         # Basic cross-compiling configuration
@@ -179,7 +181,7 @@ def configure_host_python(context):
 
         # Dependent libraries. The others are found using pkg-config: see
         # android-env.sh.
-        f"--with-openssl={prefix_dir}",
+        f"--with-openssl={prefix_subdir}",
     ]
 
     if context.args:
@@ -191,15 +193,13 @@ def make_host_python(context):
     # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so
     # delete any previous Python installation to prevent it being used during
     # the build.
-    host_dir = subdir(context.host)
-    prefix_dir = host_dir / "prefix"
-    delete_glob(f"{prefix_dir}/include/python*")
-    delete_glob(f"{prefix_dir}/lib/libpython*")
-    delete_glob(f"{prefix_dir}/lib/python*")
+    prefix_subdir = subdir(PREFIX_DIR, context.host)
+    for pattern in ("include/python*", "lib/libpython*", "lib/python*"):
+        delete_glob(f"{prefix_subdir}/{pattern}")
 
-    os.chdir(host_dir / "build")
+    os.chdir(subdir(BUILD_DIR, context.host))
     run(["make", "-j", str(os.cpu_count())], host=context.host)
-    run(["make", "install", f"prefix={prefix_dir}"], host=context.host)
+    run(["make", "install", f"prefix={prefix_subdir}"], host=context.host)
 
 
 def build_all(context):
@@ -209,8 +209,14 @@ def build_all(context):
         step(context)
 
 
+def clean(host):
+    for parent in (BUILD_DIR, PREFIX_DIR):
+        delete_glob(f"{parent}/{host}")
+
+
 def clean_all(context):
-    delete_glob(CROSS_BUILD_DIR)
+    for parent in (BUILD_DIR, PREFIX_DIR):
+        delete_glob(parent)
 
 
 def setup_sdk():

Android/testbed/app/build.gradle.kts

@@ -7,24 +7,26 @@ plugins {
 }
 
 val PYTHON_DIR = file("../../..").canonicalPath
-val PYTHON_CROSS_DIR = "$PYTHON_DIR/cross-build"
+val ANDROID_DIR = "$PYTHON_DIR/Android"
+val PREFIX_DIR = "$ANDROID_DIR/prefix"
 
 val ABIS = mapOf(
     "arm64-v8a" to "aarch64-linux-android",
     "x86_64" to "x86_64-linux-android",
-).filter { file("$PYTHON_CROSS_DIR/${it.value}").exists() }
+).filter { file("$PREFIX_DIR/${it.value}").exists() }
 if (ABIS.isEmpty()) {
     throw GradleException(
-        "No Android ABIs found in $PYTHON_CROSS_DIR: see Android/README.md " +
+        "No Android ABIs found in $PREFIX_DIR: see Android/README.md " +
         "for building instructions."
     )
 }
 
-val PYTHON_VERSION = file("$PYTHON_DIR/Include/patchlevel.h").useLines {
-    for (line in it) {
-        val match = """#define PY_VERSION\s+"(\d+\.\d+)""".toRegex().find(line)
+val PYTHON_VERSION = run {
+    val includeDir = file("$PREFIX_DIR/${ABIS.values.iterator().next()}/include")
+    for (filename in includeDir.list()!!) {
+        val match = """python(\d+\.\d+)""".toRegex().matchEntire(filename)
         if (match != null) {
-            return@useLines match.groupValues[1]
+            return@run match.groupValues[1]
         }
     }
     throw GradleException("Failed to find Python version")
@@ -55,7 +57,7 @@ android {
 
         ndk.abiFilters.addAll(ABIS.keys)
         externalNativeBuild.cmake.arguments(
-            "-DPYTHON_CROSS_DIR=$PYTHON_CROSS_DIR",
+            "-DPYTHON_PREFIX_DIR=$PREFIX_DIR",
             "-DPYTHON_VERSION=$PYTHON_VERSION",
             "-DANDROID_SUPPORT_FLEXIBLE_PAGE_SIZES=ON",
         )
@@ -136,9 +138,10 @@ androidComponents.onVariants { variant ->
     val pyPlusVer = "python$PYTHON_VERSION"
     generateTask(variant, variant.sources.assets!!) {
         into("python") {
+            // Include files such as pyconfig.h are used by some of the tests.
             into("include/$pyPlusVer") {
                 for (triplet in ABIS.values) {
-                    from("$PYTHON_CROSS_DIR/$triplet/prefix/include/$pyPlusVer")
+                    from("$PREFIX_DIR/$triplet/include/$pyPlusVer")
                 }
                 duplicatesStrategy = DuplicatesStrategy.EXCLUDE
             }
@@ -147,10 +150,10 @@ androidComponents.onVariants { variant ->
                 // To aid debugging, the source directory takes priority.
                 from("$PYTHON_DIR/Lib")
 
-                // The cross-build directory provides ABI-specific files such as
+                // The predix directory provides ABI-specific files such as
                 // sysconfigdata.
                 for (triplet in ABIS.values) {
-                    from("$PYTHON_CROSS_DIR/$triplet/prefix/lib/$pyPlusVer")
+                    from("$PREFIX_DIR/$triplet/lib/$pyPlusVer")
                 }
 
                 into("site-packages") {
@@ -166,7 +169,7 @@ androidComponents.onVariants { variant ->
     generateTask(variant, variant.sources.jniLibs!!) {
         for ((abi, triplet) in ABIS.entries) {
             into(abi) {
-                from("$PYTHON_CROSS_DIR/$triplet/prefix/lib")
+                from("$PREFIX_DIR/$triplet/lib")
                 include("libpython*.*.so")
                 include("lib*_python.so")
             }

Android/testbed/app/src/main/c/CMakeLists.txt

@@ -1,9 +1,9 @@
 cmake_minimum_required(VERSION 3.4.1)
 project(testbed)
 
-set(PREFIX_DIR ${PYTHON_CROSS_DIR}/${CMAKE_LIBRARY_ARCHITECTURE}/prefix)
-include_directories(${PREFIX_DIR}/include/python${PYTHON_VERSION})
-link_directories(${PREFIX_DIR}/lib)
+set(PYTHON_PREFIX_SUBDIR ${PYTHON_PREFIX_DIR}/${CMAKE_LIBRARY_ARCHITECTURE})
+include_directories(${PYTHON_PREFIX_SUBDIR}/include/python${PYTHON_VERSION})
+link_directories(${PYTHON_PREFIX_SUBDIR}/lib)
 link_libraries(log python${PYTHON_VERSION})
 
 add_library(main_activity SHARED main_activity.c)