Minimal fixes to make pygame bootstrap work with python2legacy (#1587)

* Force SDL and pygame recipes to be build with python2legacy instead of python2

We don't know if pygame will work for the new python2, for now we force to be build only for python2legacy because we know for sure that it works (we always can grant python2 compatibility later or not...this could be the first step to deprecate pygame bootstrap)

* Make setup_pygame.py to use python2legacy

Because we will fix the pygame bootstrap build for python2legacy not for python2

* Adapt pygame bootstrap to new python build/distribution system (make use of refactored distribution methods)

* Adapt pygame bootstrap file `Application.mk` to newer ndks

And:
  - remove hardcoded entries
  - add cflags to allow build with newer ndks
  - comment APP_STL line and set it to c++_static because the gnustl will be removed in android ndk 18, plus the default behaviour is to use the static stl lib...so...better let the android build decide depending on the ndk used.

* Fix sdl's jpeg build with newer ndks

Because with the latest changes in master branch we need an ndk with the `unified headers feature`...so...we expect to find an ndk > 15 (preferably ndk17c or ndk18b at the time of writing)...this will allow us to use the same ndk for all bootstrap builds, otherwise we should use an old ndk to build the pygame bootstrap

* Fix start.c for pygame bootstrap and reduce specific code for pygame bootstrap

* Fix ant build for pygame bootstrap

The ideal would be to move the apk build from ant to gradle...but this will require more work...for now we fix the java version problem we have with ant's build....because the other option it's far more complicated...maybe we will get there in future commits...

Author: opacam

pythonforandroid/bootstraps/common/build/jni/application/src/start.c

@@ -17,8 +17,10 @@
 #include "bootstrap_name.h"
 #ifndef BOOTSTRAP_USES_NO_SDL_HEADERS
 #include "SDL.h"
+#ifndef BOOTSTRAP_NAME_PYGAME
 #include "SDL_opengles2.h"
 #endif
+#endif
 #ifdef BOOTSTRAP_NAME_PYGAME
 #include "jniwrapperstuff.h"
 #endif
@@ -335,10 +337,8 @@ JNIEXPORT void JNICALL Java_org_kivy_android_PythonService_nativeStart(
     jobject thiz,
     jstring j_android_private,
     jstring j_android_argument,
-#if (!defined(BOOTSTRAP_NAME_PYGAME))
     jstring j_service_entrypoint,
     jstring j_python_name,
-#endif
     jstring j_python_home,
     jstring j_python_path,
     jstring j_arg) {
@@ -347,14 +347,10 @@ JNIEXPORT void JNICALL Java_org_kivy_android_PythonService_nativeStart(
       (*env)->GetStringUTFChars(env, j_android_private, &iscopy);
   const char *android_argument =
       (*env)->GetStringUTFChars(env, j_android_argument, &iscopy);
-#if (!defined(BOOTSTRAP_NAME_PYGAME))
   const char *service_entrypoint =
       (*env)->GetStringUTFChars(env, j_service_entrypoint, &iscopy);
   const char *python_name =
       (*env)->GetStringUTFChars(env, j_python_name, &iscopy);
-#else
-  const char python_name[] = "python2";
-#endif
   const char *python_home =
       (*env)->GetStringUTFChars(env, j_python_home, &iscopy);
   const char *python_path =
@@ -364,10 +360,7 @@ JNIEXPORT void JNICALL Java_org_kivy_android_PythonService_nativeStart(
   setenv("ANDROID_PRIVATE", android_private, 1);
   setenv("ANDROID_ARGUMENT", android_argument, 1);
   setenv("ANDROID_APP_PATH", android_argument, 1);
-
-#if (!defined(BOOTSTRAP_NAME_PYGAME))
   setenv("ANDROID_ENTRYPOINT", service_entrypoint, 1);
-#endif
   setenv("PYTHONOPTIMIZE", "2", 1);
   setenv("PYTHON_NAME", python_name, 1);
   setenv("PYTHONHOME", python_home, 1);

pythonforandroid/bootstraps/pygame/__init__.py

@@ -1,15 +1,14 @@
 from pythonforandroid.toolchain import Bootstrap, current_directory, info, info_main, shprint
+from pythonforandroid.util import ensure_dir
 from os.path import join, exists
-from os import walk
-import glob
 import sh
 
 
 class PygameBootstrap(Bootstrap):
     name = 'pygame'
 
-    recipe_depends = ['hostpython2', 'python2', 'pyjnius', 'sdl', 'pygame',
-                      'android', 'kivy']
+    recipe_depends = ['hostpython2legacy', 'python2legacy', 'pyjnius',
+                      'sdl', 'pygame', 'android', 'kivy']
 
     def run_distribute(self):
         info_main('# Creating Android project from build and {} bootstrap'.format(
@@ -45,53 +44,30 @@ def run_distribute(self):
                 fileh.write('sdk.dir={}'.format(self.ctx.sdk_dir))
 
             info('Copying python distribution')
-            hostpython = sh.Command(self.ctx.hostpython)
-            try:
-                shprint(hostpython, '-OO', '-m', 'compileall', self.ctx.get_python_install_dir(),
-                        _tail=10, _filterout="^Listing")
-            except sh.ErrorReturnCode:
-                pass
-            if not exists('python-install'):
-                shprint(sh.cp, '-a', self.ctx.get_python_install_dir(), './python-install')
-
-            self.distribute_libs(arch, [join(self.build_dir, 'libs', arch.arch), self.ctx.get_libs_dir(arch.arch)])
+
+            python_bundle_dir = join('_python_bundle', '_python_bundle')
+            if 'python2legacy' in self.ctx.recipe_build_order:
+                # a special case with its own packaging location
+                python_bundle_dir = 'private'
+                # And also must had an install directory, make sure of that
+                self.ctx.python_recipe.create_python_install(self.dist_dir)
+
+            self.distribute_libs(
+                arch, [join(self.build_dir, 'libs', arch.arch),
+                       self.ctx.get_libs_dir(arch.arch)])
             self.distribute_aars(arch)
             self.distribute_javaclasses(self.ctx.javaclass_dir)
 
-            info('Filling private directory')
-            if not exists(join('private', 'lib')):
-                shprint(sh.cp, '-a', join('python-install', 'lib'), 'private')
-            shprint(sh.mkdir, '-p', join('private', 'include', 'python2.7'))
-
-            shprint(sh.mv, join('libs', arch.arch, 'libpymodules.so'), 'private/')
-            shprint(sh.cp, join('python-install', 'include', 'python2.7', 'pyconfig.h'), join('private', 'include', 'python2.7/'))
-
-            info('Removing some unwanted files')
-            shprint(sh.rm, '-f', join('private', 'lib', 'libpython2.7.so'))
-            shprint(sh.rm, '-rf', join('private', 'lib', 'pkgconfig'))
-
-            with current_directory(join(self.dist_dir, 'private', 'lib', 'python2.7')):
-                # shprint(sh.xargs, 'rm', sh.grep('-E', '*\.(py|pyx|so\.o|so\.a|so\.libs)$', sh.find('.')))
-                removes = []
-                for dirname, something, filens in walk('.'):
-                    for filename in filens:
-                        for suffix in ('py', 'pyc', 'so.o', 'so.a', 'so.libs'):
-                            if filename.endswith(suffix):
-                                removes.append(filename)
-                shprint(sh.rm, '-f', *removes)
-
-                info('Deleting some other stuff not used on android')
-                # To quote the original distribute.sh, 'well...'
-                # shprint(sh.rm, '-rf', 'ctypes')
-                shprint(sh.rm, '-rf', 'lib2to3')
-                shprint(sh.rm, '-rf', 'idlelib')
-                for filename in glob.glob('config/libpython*.a'):
-                    shprint(sh.rm, '-f', filename)
-                shprint(sh.rm, '-rf', 'config/python.o')
-                shprint(sh.rm, '-rf', 'lib-dynload/_ctypes_test.so')
-                shprint(sh.rm, '-rf', 'lib-dynload/_testcapi.so')
+            ensure_dir(python_bundle_dir)
+            site_packages_dir = self.ctx.python_recipe.create_python_bundle(
+                join(self.dist_dir, python_bundle_dir), arch)
+
+            if 'sqlite3' not in self.ctx.recipe_build_order:
+                with open('blacklist.txt', 'a') as fileh:
+                    fileh.write('\nsqlite3/*\nlib-dynload/_sqlite3.so\n')
 
         self.strip_libraries(arch)
+        self.fry_eggs(site_packages_dir)
         super(PygameBootstrap, self).run_distribute()
 
 

pythonforandroid/bootstraps/pygame/build/build.py

@@ -215,6 +215,7 @@ def make_package(args):
     default_icon = 'templates/kivy-icon.png'
     default_presplash = 'templates/kivy-presplash.jpg'
     default_ouya_icon = 'templates/kivy-ouya-icon.png'
+    default_ant_properties = 'templates/ant.properties'
     # Figure out the version code, if necessary.
     if not args.numeric_version:
         for i in args.version.split('.'):
@@ -345,6 +346,9 @@ def make_package(args):
     shutil.copy(args.presplash or default_presplash,
                 'res/drawable/presplash.jpg')
 
+    # Copy over the ant.properties file.
+    shutil.copy(default_ant_properties, 'ant.properties')
+
     # If OUYA support was requested, copy over the OUYA icon
     if args.ouya_category:
         if not os.path.isdir('res/drawable-xhdpi'):

pythonforandroid/bootstraps/pygame/build/jni/Application.mk

@@ -8,8 +8,7 @@ APP_PROJECT_PATH := $(call my-dir)/..
 APP_MODULES := application sdl sdl_main tremor png jpeg freetype sdl_ttf sdl_image sqlite3
 
 APP_ABI := $(ARCH)
-# AND: I have no idea why I have to specify app_platform when distribute.sh seems to just set the sysroot cflag
-# AND: Either way, this has to *at least* be configurable
-APP_PLATFORM := android-14
-APP_STL := gnustl_static
+APP_PLATFORM := $(NDK_API)
+#APP_STL := c++_static
 APP_CFLAGS += $(OFLAG)
+APP_CFLAGS += -std=gnu89 -Wno-return-type

pythonforandroid/bootstraps/pygame/build/src/org/renpy/android/SDLSurfaceView.java

@@ -707,6 +707,8 @@ public void run() {
         nativeInitJavaCallbacks();
         nativeSetEnv("ANDROID_PRIVATE", mFilesDirectory);
         nativeSetEnv("ANDROID_UNPACK", mFilesDirectory);
+        nativeSetEnv("ANDROID_ENTRYPOINT", mArgument + "/main.py");
+        nativeSetEnv("PYTHON_NAME", "python");
         nativeSetEnv("ANDROID_ARGUMENT", mArgument);
         nativeSetEnv("ANDROID_APP_PATH", mArgument);
         nativeSetEnv("PYTHONOPTIMIZE", "2");

pythonforandroid/bootstraps/pygame/build/templates/ant.properties

@@ -0,0 +1,2 @@
+java.target=1.7
+java.source=1.7

pythonforandroid/recipes/pygame/__init__.py

@@ -12,7 +12,7 @@ class PygameRecipe(Recipe):
     version = '1.9.1'
     url = 'http://pygame.org/ftp/pygame-{version}release.tar.gz'
 
-    depends = ['python2', 'sdl']
+    depends = ['python2legacy', 'sdl']
     conflicts = ['sdl2']
 
     patches = ['patches/fix-surface-access.patch',

pythonforandroid/recipes/pygame_bootstrap_components/__init__.py

@@ -25,5 +25,11 @@ def prebuild_arch(self, arch):
         info('Unpacking was successful, deleting original container dir')
         shprint(sh.rm, '-rf', self.get_build_dir(arch))
 
+        info('Applying jpeg assembler patch')
+        ndk_15_plus_patch = join(self.get_recipe_dir(), 'jpeg-ndk15-plus.patch')
+        shprint(sh.patch, '-t', '-d',
+                join(self.get_build_container_dir(arch), 'jpeg'), '-p1',
+                '-i', ndk_15_plus_patch, _tail=10)
+
 
 recipe = PygameJNIComponentsRecipe()

pythonforandroid/recipes/pygame_bootstrap_components/jpeg-ndk15-plus.patch

@@ -0,0 +1,44 @@
+The distributed jpeg has troubles to be build with newer ndks, starting from
+the introduction of the `unified headers` (ndk > 15). This patch allow us to
+build the distributed `external jpeg` in sdl package, got the solution in here:
+https://github.com/oNaiPs/droidVncServer/issues/53
+--- jpeg/Android.mk.orig	2015-06-21 15:14:54.000000000 +0200
++++ jpeg/Android.mk	2019-01-14 10:57:06.384806168 +0100
+@@ -20,7 +20,7 @@
+ endif
+
+ # temp fix until we understand why this broke cnn.com
+-#ANDROID_JPEG_NO_ASSEMBLER := true
++ANDROID_JPEG_NO_ASSEMBLER := true
+
+ ifeq ($(strip $(ANDROID_JPEG_NO_ASSEMBLER)),true)
+ LOCAL_SRC_FILES += jidctint.c jidctfst.c
+ --- jpeg/jidctfst.S.orig	2019-01-14 11:00:38.000000000 +0100
++++ jpeg/jidctfst.S	2019-01-14 11:00:56.844803970 +0100
+@@ -63,7 +63,7 @@
+
+
+ jpeg_idct_ifast:
+-    PLD     [r2, #0]
++    pld     [r2, #0]
+     stmdb   sp!, {r4,r5, r6,r7, r8,r9, r10,r11, r12,lr}
+     ldr     r4, [sp, #4*10]
+     sub     sp, #local_SIZE
+@@ -256,7 +256,7 @@
+
+ HLoopStart:
+     // reset pointers
+-    PLD     [sp, #off_WORKSPACE]
++    pld     [sp, #off_WORKSPACE]
+     add     ip, sp, #off_WORKSPACE
+     ldr     r10, local_RANGE_TABLE
+
+@@ -268,7 +268,7 @@
+     str      r0, local_OUTPUT_BUF
+     add      fp, r2, r1
+
+-    PLD      [ip, #32]
++    pld      [ip, #32]
+     ldmia    ip!, {r0-r7}
+
+     cmp      r1, #0

pythonforandroid/recipes/sdl/__init__.py

@@ -8,7 +8,7 @@ class LibSDLRecipe(BootstrapNDKRecipe):
     version = "1.2.14"
     url = None
     name = 'sdl'
-    depends = ['python2', 'pygame_bootstrap_components']
+    depends = ['python2legacy', 'pygame_bootstrap_components']
     conflicts = ['sdl2']
 
     def build_arch(self, arch):