Rework for Pillow/pil recipes & update jpeg and png (#1573)

opacam · AndreMiras · commit e3123f414a4c · 2019-01-13T23:45:00.000+01:00
* Force to build only static library for freetype/harfbuzz, format source code and fix imports

We force to only build the static library to avoid that Pillow/pil gets linked with the shared library, plus we cannot use those libraries in his shared form because we will have conflicts with the ones distributed by android os...so no need to build those libraries unless properly versioned as we do with openssl libs

* Update png version

* Update jpeg and move to mainline

Also add a patch that may allow us to build the libraries in his shared form...in case we need them

* Rework Pillow

- Fix compatibility for new jpeg
- add freetype/harfbuzz support
- move libraries from LDFLAGS to LIBS
- format source code

* Rework pil

- Fix compatibility for new jpeg
- add freetype/harfbuzz support
- move libraries from LDFLAGS to LIBS
- make it work with python2 and python2legacy

* Add cmake reminder to quickstart.rst

The recent rework of the jpeg recipe has introduced a new build dependency for us, so we add it to the section `Installing Dependencies` to no forgot about it.

* Fix hardcoded url for png

We set the last commit published for a version because the author of the github's repo never released/tagged it, despite He performed the necessary changes in master branch and notify about it in the README file...so this way we don't hardcode the url.

Note: we should try to move the png repo to mainline because of  the issue mentioned above and the fact that the last commit of the repo is more than one year old
diff --git a/doc/source/quickstart.rst b/doc/source/quickstart.rst
@@ -63,6 +63,7 @@ p4a has several dependencies that must be installed:
 - ccache (optional)
 - autoconf (for ffpyplayer_codecs recipe)
 - libtool (for ffpyplayer_codecs recipe)
+- cmake (required for some native code recipes like jpeg's recipe)
 
 On recent versions of Ubuntu and its derivatives you may be able to
 install most of these with::
diff --git a/pythonforandroid/recipes/Pillow/__init__.py b/pythonforandroid/recipes/Pillow/__init__.py
@@ -1,72 +1,58 @@
 from pythonforandroid.recipe import CompiledComponentsPythonRecipe
-from pythonforandroid.toolchain import shprint
-from os.path import join, dirname
-import sh
+from os.path import join
 
 
 class PillowRecipe(CompiledComponentsPythonRecipe):
 
     version = '5.2.0'
     url = 'https://github.com/python-pillow/Pillow/archive/{version}.tar.gz'
     site_packages_name = 'Pillow'
-    depends = [
-        'png',
-        'jpeg',
-        'freetype',
-        'setuptools'
-    ]
-    patches = [
-        join('patches', 'fix-docstring.patch'),
-        join('patches', 'fix-setup.patch')
-    ]
+    depends = ['png', 'jpeg', 'freetype', 'setuptools']
+    patches = [join('patches', 'fix-docstring.patch'),
+               join('patches', 'fix-setup.patch')]
 
     call_hostpython_via_targetpython = False
 
-    def get_recipe_env(self, arch=None):
-        env = super(PillowRecipe, self).get_recipe_env(arch)
-        py_ver = self.ctx.python_recipe.version[0:3]
+    def get_recipe_env(self, arch=None, with_flags_in_cc=True):
+        env = super(PillowRecipe, self).get_recipe_env(arch, with_flags_in_cc)
 
-        ndk_dir = self.ctx.ndk_platform
-        ndk_lib_dir = join(ndk_dir, 'usr', 'lib')
-        ndk_include_dir = (
-            join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
-            if py_ver == '2.7' else join(ndk_dir, 'usr', 'include'))
+        env['ANDROID_ROOT'] = join(self.ctx.ndk_platform, 'usr')
+        ndk_lib_dir = join(self.ctx.ndk_platform, 'usr', 'lib')
+        ndk_include_dir = join(self.ctx.ndk_dir, 'sysroot', 'usr', 'include')
 
         png = self.get_recipe('png', self.ctx)
         png_lib_dir = png.get_lib_dir(arch)
         png_jni_dir = png.get_jni_dir(arch)
 
         jpeg = self.get_recipe('jpeg', self.ctx)
-        jpeg_lib_dir = jpeg.get_lib_dir(arch)
-        jpeg_jni_dir = jpeg.get_jni_dir(arch)
+        jpeg_inc_dir = jpeg_lib_dir = jpeg.get_build_dir(arch.arch)
 
-        env['JPEG_ROOT'] = '{}|{}'.format(jpeg_lib_dir, jpeg_jni_dir)
+        freetype = self.get_recipe('freetype', self.ctx)
+        free_lib_dir = join(freetype.get_build_dir(arch.arch), 'objs', '.libs')
+        free_inc_dir = join(freetype.get_build_dir(arch.arch), 'include')
+
+        # harfbuzz is a direct dependency of freetype and we need the proper
+        # flags to successfully build the Pillow recipe, so we add them here.
+        harfbuzz = self.get_recipe('harfbuzz', self.ctx)
+        harf_lib_dir = join(harfbuzz.get_build_dir(arch.arch), 'src', '.libs')
+        harf_inc_dir = harfbuzz.get_build_dir(arch.arch)
+
+        env['JPEG_ROOT'] = '{}|{}'.format(jpeg_lib_dir, jpeg_inc_dir)
+        env['FREETYPE_ROOT'] = '{}|{}'.format(free_lib_dir, free_inc_dir)
         env['ZLIB_ROOT'] = '{}|{}'.format(ndk_lib_dir, ndk_include_dir)
 
-        cflags = ' -nostdinc'
-        cflags += ' -I{} -L{}'.format(png_jni_dir, png_lib_dir)
-        cflags += ' -I{} -L{}'.format(jpeg_jni_dir, jpeg_lib_dir)
-        cflags += ' -I{} -L{}'.format(ndk_include_dir, ndk_lib_dir)
-
-        gcc_lib = shprint(
-            sh.gcc, '-print-libgcc-file-name').stdout.decode('utf-8').split('\n')[0]
-        gcc_include = join(dirname(gcc_lib), 'include')
-        cflags += ' -I{}'.format(gcc_include)
-
-        if self.ctx.ndk == 'crystax':
-            py_inc_dir = join(
-                self.ctx.ndk_dir, 'sources', 'python', py_ver, 'include', 'python')
-            py_lib_dir = join(
-                self.ctx.ndk_dir, 'sources', 'python', py_ver, 'libs', arch.arch)
-            cflags += ' -I{}'.format(py_inc_dir)
-            env['LDFLAGS'] += ' -L{} -lpython{}m'.format(py_lib_dir, py_ver)
-
-        env['LDFLAGS'] += ' {} -L{}'.format(env['CFLAGS'], self.ctx.libs_dir)
+        cflags = ' -I{}'.format(png_jni_dir)
+        cflags += ' -I{} -I{}'.format(harf_inc_dir, join(harf_inc_dir, 'src'))
+        cflags += ' -I{}'.format(free_inc_dir)
+        cflags += ' -I{}'.format(jpeg_inc_dir)
+        cflags += ' -I{}'.format(ndk_include_dir)
+
+        env['LIBS'] = ' -lpng -lfreetype -lharfbuzz -ljpeg -lturbojpeg'
+
+        env['LDFLAGS'] += ' -L{} -L{} -L{} -L{}'.format(
+            png_lib_dir, harf_lib_dir, jpeg_lib_dir, ndk_lib_dir)
         if cflags not in env['CFLAGS']:
             env['CFLAGS'] += cflags
-        env['LDSHARED'] = '{} {}'.format(
-            env['CC'],
-            '-pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions')
         return env
 
 
diff --git a/pythonforandroid/recipes/freetype/__init__.py b/pythonforandroid/recipes/freetype/__init__.py
@@ -1,18 +1,20 @@
-
-from pythonforandroid.toolchain import Recipe, current_directory, shprint
+from pythonforandroid.toolchain import Recipe
+from pythonforandroid.util import current_directory
+from pythonforandroid.logger import shprint
 from os.path import exists, join, realpath
 import sh
 
 
 class FreetypeRecipe(Recipe):
 
     version = '2.5.5'
-    url = 'http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz'
+    url = 'http://download.savannah.gnu.org/releases/freetype/freetype-{version}.tar.gz'  # noqa
 
     depends = ['harfbuzz']
 
     def should_build(self, arch):
-        if exists(join(self.get_build_dir(arch.arch), 'objs', '.libs', 'libfreetype.so')):
+        if exists(join(self.get_build_dir(arch.arch),
+                       'objs', '.libs', 'libfreetype.a')):
             return False
         return True
 
@@ -22,17 +24,21 @@ def build_arch(self, arch):
         harfbuzz_recipe = Recipe.get_recipe('harfbuzz', self.ctx)
         env['LDFLAGS'] = ' '.join(
             [env['LDFLAGS'],
-             '-L{}'.format(join(harfbuzz_recipe.get_build_dir(arch.arch), 'src', '.libs'))])
+             '-L{}'.format(join(harfbuzz_recipe.get_build_dir(arch.arch),
+                                'src', '.libs'))])
 
         with current_directory(self.get_build_dir(arch.arch)):
             configure = sh.Command('./configure')
-            shprint(configure, '--host=arm-linux-androideabi',
+            shprint(configure,
+                    '--host=arm-linux-androideabi',
                     '--prefix={}'.format(realpath('.')),
-                    '--without-zlib', '--with-png=no', '--enable-shared',
+                    '--without-zlib',
+                    '--with-png=no',
+                    '--disable-shared',
                     _env=env)
             shprint(sh.make, '-j5', _env=env)
 
-            shprint(sh.cp, 'objs/.libs/libfreetype.so', self.ctx.libs_dir)
+            shprint(sh.cp, 'objs/.libs/libfreetype.a', self.ctx.libs_dir)
 
 
 recipe = FreetypeRecipe()
diff --git a/pythonforandroid/recipes/harfbuzz/__init__.py b/pythonforandroid/recipes/harfbuzz/__init__.py
@@ -1,15 +1,17 @@
-
-from pythonforandroid.toolchain import Recipe, current_directory, shprint
+from pythonforandroid.toolchain import Recipe
+from pythonforandroid.util import current_directory
+from pythonforandroid.logger import shprint
 from os.path import exists, join
 import sh
 
 
 class HarfbuzzRecipe(Recipe):
     version = '0.9.40'
-    url = 'http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-{version}.tar.bz2'
+    url = 'http://www.freedesktop.org/software/harfbuzz/release/harfbuzz-{version}.tar.bz2'  # noqa
 
     def should_build(self, arch):
-        if exists(join(self.get_build_dir(arch.arch), 'src', '.libs', 'libharfbuzz.so')):
+        if exists(join(self.get_build_dir(arch.arch),
+                       'src', '.libs', 'libharfbuzz.a')):
             return False
         return True
 
@@ -22,11 +24,16 @@ def build_arch(self, arch):
         with current_directory(self.get_build_dir(arch.arch)):
             configure = sh.Command('./configure')
             shprint(configure, '--without-icu', '--host=arm-linux=androideabi',
-                    '--prefix={}'.format(join(self.ctx.build_dir, 'python-install')),
-                    '--without-freetype', '--without-glib', _env=env)
+                    '--prefix={}'.format(
+                        join(self.ctx.build_dir, 'python-install')),
+                    '--without-freetype',
+                    '--without-glib',
+                    '--disable-shared',
+                    _env=env)
             shprint(sh.make, '-j5', _env=env)
 
-            shprint(sh.cp, '-L', join('src', '.libs', 'libharfbuzz.so'), self.ctx.libs_dir)
+            shprint(sh.cp, '-L', join('src', '.libs', 'libharfbuzz.a'),
+                    self.ctx.libs_dir)
 
 
 recipe = HarfbuzzRecipe()
diff --git a/pythonforandroid/recipes/jpeg/__init__.py b/pythonforandroid/recipes/jpeg/__init__.py
@@ -1,35 +1,78 @@
-from pythonforandroid.recipe import NDKRecipe
+from pythonforandroid.recipe import Recipe
 from pythonforandroid.logger import shprint
 from pythonforandroid.util import current_directory
 from os.path import join, exists
+from os import environ, uname
+from glob import glob
 import sh
 
 
-class JpegRecipe(NDKRecipe):
+class JpegRecipe(Recipe):
+    '''
+    .. versionchanged:: 0.6.0
+        rewrote recipe to be build with clang and updated libraries to latest
+        version of the official git repo.
+    '''
     name = 'jpeg'
-    version = 'linaro-android'
-    url = 'git+https://git.linaro.org/people/tomgall/libjpeg-turbo/libjpeg-turbo'
+    version = '2.0.1'
+    url = 'https://github.com/libjpeg-turbo/libjpeg-turbo/archive/{version}.tar.gz'  # noqa
+    # we will require this below patch to build the shared library
+    # patches = ['remove-version.patch']
 
-    patches = ['build-static.patch']
+    def should_build(self, arch):
+        return not exists(join(self.get_build_dir(arch.arch),
+                               'libturbojpeg.a'))
 
-    generated_libraries = ['libjpeg.a']
+    def build_arch(self, arch):
+        super(JpegRecipe, self).build_arch(arch)
+        build_dir = self.get_build_dir(arch.arch)
 
-    def prebuild_arch(self, arch):
-        super(JpegRecipe, self).prebuild_arch(arch)
+        # TODO: Fix simd/neon
+        with current_directory(build_dir):
+            env = self.get_recipe_env(arch)
+            toolchain_file = join(self.ctx.ndk_dir,
+                                  'build/cmake/android.toolchain.cmake')
 
-        build_dir = self.get_build_dir(arch.arch)
-        app_mk = join(build_dir, 'Application.mk')
-        if not exists(app_mk):
-            shprint(sh.cp, join(self.get_recipe_dir(), 'Application.mk'), app_mk)
-        jni_ln = join(build_dir, 'jni')
-        if not exists(jni_ln):
-            shprint(sh.ln, '-s', build_dir, jni_ln)
+            shprint(sh.rm, '-f', 'CMakeCache.txt', 'CMakeFiles/')
+            shprint(sh.cmake, '-G', 'Unix Makefiles',
+                    '-DCMAKE_SYSTEM_NAME=Android',
+                    '-DCMAKE_SYSTEM_PROCESSOR={cpu}'.format(cpu='arm'),
+                    '-DCMAKE_POSITION_INDEPENDENT_CODE=1',
+                    '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch),
+                    '-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir,
+                    '-DCMAKE_C_COMPILER={toolchain}/bin/clang'.format(
+                        toolchain=env['TOOLCHAIN']),
+                    '-DCMAKE_CXX_COMPILER={toolchain}/bin/clang++'.format(
+                        toolchain=env['TOOLCHAIN']),
+                    '-DCMAKE_BUILD_TYPE=Release',
+                    '-DCMAKE_INSTALL_PREFIX=./install',
+                    '-DCMAKE_TOOLCHAIN_FILE=' + toolchain_file,
 
-    def build_arch(self, arch):
-        super(JpegRecipe, self).build_arch(arch)
-        with current_directory(self.get_lib_dir(arch)):
-            shprint(sh.mv, 'libjpeg.a', 'libjpeg-orig.a')
-            shprint(sh.ar, '-rcT', 'libjpeg.a', 'libjpeg-orig.a', 'libsimd.a')
+                    '-DANDROID_ABI={arch}'.format(arch=arch.arch),
+                    '-DANDROID_ARM_NEON=ON',
+                    '-DENABLE_NEON=ON',
+                    # '-DREQUIRE_SIMD=1',
+
+                    # Force disable shared, with the static ones is enough
+                    '-DENABLE_SHARED=0',
+                    '-DENABLE_STATIC=1',
+                    _env=env)
+            shprint(sh.make, _env=env)
+
+            # copy static libs to libs collection
+            for lib in glob(join(build_dir, '*.a')):
+                shprint(sh.cp, '-L', lib, self.ctx.libs_dir)
+
+    def get_recipe_env(self, arch=None, with_flags_in_cc=False, clang=True):
+        env = environ.copy()
+
+        build_platform = '{system}-{machine}'.format(
+            system=uname()[0], machine=uname()[-1]).lower()
+        env['TOOLCHAIN'] = join(self.ctx.ndk_dir, 'toolchains/llvm/'
+                                'prebuilt/{build_platform}'.format(
+                                    build_platform=build_platform))
+
+        return env
 
 
 recipe = JpegRecipe()
diff --git a/pythonforandroid/recipes/jpeg/remove-version.patch b/pythonforandroid/recipes/jpeg/remove-version.patch
@@ -0,0 +1,12 @@
+--- jpeg/CMakeLists.txt.orig	2018-11-12 20:20:28.000000000 +0100
++++ jpeg/CMakeLists.txt	2018-12-14 12:43:45.338704504 +0100
+@@ -573,6 +573,9 @@
+     add_library(turbojpeg SHARED ${TURBOJPEG_SOURCES})
+     set_property(TARGET turbojpeg PROPERTY COMPILE_FLAGS
+       "-DBMP_SUPPORTED -DPPM_SUPPORTED")
++    set_property(TARGET jpeg PROPERTY NO_SONAME 1)
++    set_property(TARGET turbojpeg PROPERTY NO_SONAME 1)
++    set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "")
+     if(WIN32)
+       set_target_properties(turbojpeg PROPERTIES DEFINE_SYMBOL DLLDEFINE)
+     endif()
diff --git a/pythonforandroid/recipes/pil/__init__.py b/pythonforandroid/recipes/pil/__init__.py
diff --git a/pythonforandroid/recipes/pil/fix-directories.patch b/pythonforandroid/recipes/pil/fix-directories.patch
diff --git a/pythonforandroid/recipes/png/__init__.py b/pythonforandroid/recipes/png/__init__.py
