From 82f5e02b0bf874fb029b7e54731728bbf573ddb3 Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Tue, 14 May 2024 19:44:32 +0530 Subject: [PATCH 1/4] recipes: add `uvloop` recipe --- .../recipes/libpthread/__init__.py | 51 +++++++++++++++++++ pythonforandroid/recipes/uvloop/__init__.py | 17 +++++++ 2 files changed, 68 insertions(+) create mode 100644 pythonforandroid/recipes/libpthread/__init__.py create mode 100644 pythonforandroid/recipes/uvloop/__init__.py diff --git a/pythonforandroid/recipes/libpthread/__init__.py b/pythonforandroid/recipes/libpthread/__init__.py new file mode 100644 index 0000000000..10feca475a --- /dev/null +++ b/pythonforandroid/recipes/libpthread/__init__.py @@ -0,0 +1,51 @@ +from os import makedirs, remove +from os.path import exists, join +import sh + +from pythonforandroid.recipe import Recipe +from pythonforandroid.logger import shprint + + +class LibPthread(Recipe): + ''' + This is a dumb recipe. We may need this because some recipes inserted some + flags `-lpthread` without our control, case of: + + - :class:`~pythonforandroid.recipes.uvloop.UvloopRecipe` + + .. note:: the libpthread doesn't exist in android but it is integrated into + libc, so we create a symbolic link which we will remove when our build + finishes''' + + def build_arch(self, arch): + libc_path = join(arch.ndk_lib_dir_versioned, 'libc') + # Create a temporary folder to add to link path with a fake libpthread.so: + fake_libpthread_temp_folder = join( + self.get_build_dir(arch.arch), + "p4a-libpthread-recipe-tempdir" + ) + if not exists(fake_libpthread_temp_folder): + makedirs(fake_libpthread_temp_folder) + + # Set symlinks, and make sure to update them on every build run: + if exists(join(fake_libpthread_temp_folder, "libpthread.so")): + remove(join(fake_libpthread_temp_folder, "libpthread.so")) + shprint(sh.ln, '-sf', + libc_path + '.so', + join(fake_libpthread_temp_folder, "libpthread.so"), + ) + if exists(join(fake_libpthread_temp_folder, "libpthread.a")): + remove(join(fake_libpthread_temp_folder, "libpthread.a")) + shprint(sh.ln, '-sf', + libc_path + '.a', + join(fake_libpthread_temp_folder, "libpthread.a"), + ) + + # Add folder as -L link option for all recipes if not done yet: + if fake_libpthread_temp_folder not in arch.extra_global_link_paths: + arch.extra_global_link_paths.append( + fake_libpthread_temp_folder + ) + + +recipe = LibPthread() diff --git a/pythonforandroid/recipes/uvloop/__init__.py b/pythonforandroid/recipes/uvloop/__init__.py new file mode 100644 index 0000000000..fcb2204422 --- /dev/null +++ b/pythonforandroid/recipes/uvloop/__init__.py @@ -0,0 +1,17 @@ +from pythonforandroid.recipe import PyProjectRecipe + + +class UvloopRecipe(PyProjectRecipe): + # 0.19.0 + version = '6c770dc3fbdd281d15c2ad46588c139696f9269c' + url = 'git+https://github.com/MagicStack/uvloop' + depends = ['librt', 'libpthread'] + + def get_recipe_env(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) + env["LIBUV_CONFIGURE_HOST"] = arch.command_prefix + env["PLATFORM"] = "android" + return env + + +recipe = UvloopRecipe() From 8cf51bfe8398794f6794109adf4083dcd10550cb Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Fri, 17 May 2024 17:13:46 +0530 Subject: [PATCH 2/4] recipes: add `pyreqwest_impersonate` recipe --- .../recipes/pyreqwest_impersonate/__init__.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 pythonforandroid/recipes/pyreqwest_impersonate/__init__.py diff --git a/pythonforandroid/recipes/pyreqwest_impersonate/__init__.py b/pythonforandroid/recipes/pyreqwest_impersonate/__init__.py new file mode 100644 index 0000000000..7e8d5db9ae --- /dev/null +++ b/pythonforandroid/recipes/pyreqwest_impersonate/__init__.py @@ -0,0 +1,33 @@ +from pythonforandroid.logger import info +from pythonforandroid.recipe import RustCompiledComponentsRecipe + + +class Pyreqwest_impersonateRecipe(RustCompiledComponentsRecipe): + version = "v0.4.5" + url = "https://github.com/deedy5/pyreqwest_impersonate/archive/refs/tags/{version}.tar.gz" + + def get_recipe_env_post(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) + env["ANDROID_NDK_HOME"] = self.ctx.ndk.llvm_prebuilt_dir + return env + + def get_recipe_env_pre(self, arch, **kwargs): + env = super().get_recipe_env(arch, **kwargs) + env["ANDROID_NDK_HOME"] = self.ctx.ndk_dir + return env + + def build_arch(self, arch): + # Why need of two env? + # Because there are two dependencies which accepts + # different ANDROID_NDK_HOME + self.get_recipe_env = self.get_recipe_env_pre + prebuild_ = super().build_arch + try: + prebuild_(arch) + except Exception: + info("pyreqwest_impersonate first build failed, as expected") + self.get_recipe_env = self.get_recipe_env_post + prebuild_(arch) + + +recipe = Pyreqwest_impersonateRecipe() From 6bf7513ef80a221fc47da235a98c064b4e1510bc Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Fri, 17 May 2024 16:22:12 +0530 Subject: [PATCH 3/4] recipes: add `panda3d` recipe --- pythonforandroid/recipe.py | 8 +- pythonforandroid/recipes/panda3d/__init__.py | 72 +++++++++++ .../recipes/panda3d/makepanda.patch | 113 ++++++++++++++++++ 3 files changed, 188 insertions(+), 5 deletions(-) create mode 100644 pythonforandroid/recipes/panda3d/__init__.py create mode 100644 pythonforandroid/recipes/panda3d/makepanda.patch diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index c4131c53e6..aa10434525 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1204,8 +1204,8 @@ def get_wheel_platform_tag(self, arch): "x86": "i686", }[arch.arch] - def install_wheel(self, arch, built_wheels): - _wheel = built_wheels[0] + def install_wheel(self, arch, pattern): + _wheel = [realpath(whl) for whl in glob.glob(pattern)][0] built_wheel_dir = dirname(_wheel) # Fix wheel platform tag wheel_tag = wheel_tags( @@ -1247,13 +1247,11 @@ def build_arch(self, arch): "builddir={}".format(sub_build_dir), ] + self.extra_build_args - built_wheels = [] with current_directory(build_dir): shprint( sh.Command(self.ctx.python_recipe.python_exe), *build_args, _env=env ) - built_wheels = [realpath(whl) for whl in glob.glob("dist/*.whl")] - self.install_wheel(arch, built_wheels) + self.install_wheel(arch, join(build_dir, "dist", "*.whl")) class MesonRecipe(PyProjectRecipe): diff --git a/pythonforandroid/recipes/panda3d/__init__.py b/pythonforandroid/recipes/panda3d/__init__.py new file mode 100644 index 0000000000..2a1ea96ea1 --- /dev/null +++ b/pythonforandroid/recipes/panda3d/__init__.py @@ -0,0 +1,72 @@ +import sh +from os.path import join, basename +from glob import glob +from multiprocessing import cpu_count +from pythonforandroid.recipe import PyProjectRecipe +from pythonforandroid.util import current_directory, ensure_dir +from pythonforandroid.logger import shprint + + +class Panda3dRecipe(PyProjectRecipe): + version = "1.10.14" + url = "https://github.com/panda3d/panda3d/archive/refs/tags/v{version}.tar.gz" + patches = ["makepanda.patch"] + + def get_recipe_env(self, arch): + env = super().get_recipe_env(arch) + env["ANDROID_NDK_ROOT"] = self.ctx.ndk_dir + env["ANDROID_SDK_ROOT"] = self.ctx.sdk_dir + env["ANDROID_TARGET_API"] = str(self.ctx.android_api) + + env["CXXFLAGS"] += " -stdlib=libstdc++ -Wno-unsupported-floating-point-opt" + env["CFLAGS"] += " -Wno-unsupported-floating-point-opt" + + # Python includes + python_includes = self.ctx.python_recipe.include_root(arch.arch) + env["CXXFLAGS"] += " -I{}".format(python_includes) + env["CFLAGS"] += " -I{}".format(python_includes) + + return env + + def build_arch(self, arch): + self.install_hostpython_prerequisites() + build_dir = self.get_build_dir(arch) + env = self.get_recipe_env(arch) + outputdir = join(build_dir, "dist") + ensure_dir(outputdir) + # Used by makepanda + _arch = { + "armeabi-v7a": "armv7a", + "arm64-v8a": "aarch64", + "x86": "x86", + "x86_64": "x86_64", + }[arch.arch] + + # Setup python lib folder + panda3d_lib_dir = join(build_dir, "thirdparty/android-libs-{}".format(_arch), "python", "lib") + ensure_dir(panda3d_lib_dir) + for lib in glob(join(self.ctx.python_recipe.link_root(arch.arch), "*.so")): + shprint( + sh.ln, "-sf", lib, join(panda3d_lib_dir, basename(lib)) + ) + + with current_directory(build_dir): + shprint( + sh.Command(self.hostpython_location), + "makepanda/makepanda.py", + "--everything", + "--outputdir", + outputdir, + "--arch", + _arch, + "--target", + "android-{}".format(self.ctx.ndk_api), + "--threads", + str(cpu_count()), + "--wheel", + _env=env + ) + self.install_wheel(arch, join(build_dir, "dist", "*.whl")) + + +recipe = Panda3dRecipe() diff --git a/pythonforandroid/recipes/panda3d/makepanda.patch b/pythonforandroid/recipes/panda3d/makepanda.patch new file mode 100644 index 0000000000..b33305a065 --- /dev/null +++ b/pythonforandroid/recipes/panda3d/makepanda.patch @@ -0,0 +1,113 @@ +diff '--color=auto' -uNr panda3d/makepanda/makepandacore.py panda3d.mod/makepanda/makepandacore.py +--- panda3d/makepanda/makepandacore.py 2024-05-17 15:33:46.876526187 +0530 ++++ panda3d.mod/makepanda/makepandacore.py 2024-05-17 17:06:05.318793873 +0530 +@@ -2681,15 +2681,17 @@ + arch_dir = 'arch-arm64' + else: + arch_dir = 'arch-' + arch +- SDK["SYSROOT"] = os.path.join(ndk_root, 'platforms', 'android-%s' % (api), arch_dir).replace('\\', '/') +- #IncDirectory("ALWAYS", os.path.join(SDK["SYSROOT"], 'usr', 'include')) ++ SDK["SYSROOT"] = os.path.join(ndk_root, "toolchains", "llvm", "prebuilt", "linux-x86_64", "sysroot", "usr") ++ IncDirectory("ALWAYS", os.path.join(SDK["SYSROOT"], 'usr', 'include')) + + # Starting with NDK r16, libc++ is the recommended STL to use. + stdlibc = os.path.join(ndk_root, 'sources', 'cxx-stl', 'llvm-libc++') + IncDirectory("ALWAYS", os.path.join(stdlibc, 'include').replace('\\', '/')) +- LibDirectory("ALWAYS", os.path.join(stdlibc, 'libs', abi).replace('\\', '/')) ++ LibDirectory("ALWAYS", os.path.join(ndk_root, "toolchains", "llvm", "prebuilt", ++ "linux-x86_64", "sysroot", "usr", "lib", ++ ANDROID_TRIPLE, str(ANDROID_API)) .replace('\\', '", "')) + +- stl_lib = os.path.join(stdlibc, 'libs', abi, 'libc++_shared.so') ++ stl_lib = os.path.join(ndk_root, SDK["SYSROOT"], "lib", ANDROID_TRIPLE, "libc++_shared.so") + LibName("ALWAYS", stl_lib.replace('\\', '/')) + CopyFile(os.path.join(GetOutputDir(), 'lib', 'libc++_shared.so'), stl_lib) + +@@ -2699,11 +2701,12 @@ + IncDirectory("ALWAYS", support.replace('\\', '/')) + if api < 21: + LibName("ALWAYS", "-landroid_support") +- ++ ++ target_api = os.environ["ANDROID_TARGET_API"] + # Determine the location of android.jar. +- SDK["ANDROID_JAR"] = os.path.join(sdk_root, 'platforms', 'android-%s' % (api), 'android.jar') ++ SDK["ANDROID_JAR"] = os.path.join(sdk_root, 'platforms', 'android-%s' % (target_api), 'android.jar') + if not os.path.isfile(SDK["ANDROID_JAR"]): +- exit("Cannot find %s. Install platform API level %s via the SDK manager or change the targeted API level with --target=android-#" % (SDK["ANDROID_JAR"], api)) ++ exit("Cannot find %s. Install platform API level %s via the SDK manager or change the targeted API level with --target=android-#" % (SDK["ANDROID_JAR"], target_api)) + + # Which build tools versions do we have? Pick the latest. + versions = [] +diff '--color=auto' -uNr panda3d/makepanda/makepanda.py panda3d.mod/makepanda/makepanda.py +--- panda3d/makepanda/makepanda.py 2024-05-17 15:33:46.856521623 +0530 ++++ panda3d.mod/makepanda/makepanda.py 2024-05-17 16:53:32.974433320 +0530 +@@ -1197,7 +1197,7 @@ + # CgGL is covered by the Cg framework, and we don't need X11 components on OSX + if not PkgSkip("NVIDIACG") and not RUNTIME: + SmartPkgEnable("CGGL", "", ("CgGL"), "Cg/cgGL.h", thirdparty_dir = "nvidiacg") +- if not RUNTIME and GetTarget() != "android": ++ if not RUNTIME and False: # GetTarget() != "android": + SmartPkgEnable("X11", "x11", "X11", ("X11", "X11/Xlib.h", "X11/XKBlib.h")) + + if GetHost() != "darwin": +@@ -1580,23 +1580,12 @@ + if 'NOARCH:' + arch.upper() not in opts: + cmd += " -arch %s" % arch + +- if "SYSROOT" in SDK: +- if GetTarget() != "android": +- cmd += ' --sysroot=%s' % (SDK["SYSROOT"]) +- else: +- ndk_dir = SDK["ANDROID_NDK"].replace('\\', '/') +- cmd += ' -isystem %s/sysroot/usr/include' % (ndk_dir) +- cmd += ' -isystem %s/sysroot/usr/include/%s' % (ndk_dir, SDK["ANDROID_TRIPLE"]) +- cmd += ' -no-canonical-prefixes' +- + # Android-specific flags. + arch = GetTargetArch() + + if GetTarget() == "android": + # Most of the specific optimization flags here were + # just copied from the default Android Makefiles. +- if "ANDROID_API" in SDK: +- cmd += ' -D__ANDROID_API__=' + str(SDK["ANDROID_API"]) + if "ANDROID_GCC_TOOLCHAIN" in SDK: + cmd += ' -gcc-toolchain ' + SDK["ANDROID_GCC_TOOLCHAIN"].replace('\\', '/') + cmd += ' -ffunction-sections -funwind-tables' +@@ -2117,33 +2106,17 @@ + for arch in OSX_ARCHS: + if 'NOARCH:' + arch.upper() not in opts: + cmd += " -arch %s" % arch +- ++ + elif GetTarget() == 'android': + arch = GetTargetArch() + if "ANDROID_GCC_TOOLCHAIN" in SDK: + cmd += ' -gcc-toolchain ' + SDK["ANDROID_GCC_TOOLCHAIN"].replace('\\', '/') + cmd += " -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now" +- if arch == 'armv7a': +- cmd += ' -target armv7-none-linux-androideabi' +- cmd += " -march=armv7-a -Wl,--fix-cortex-a8" +- elif arch == 'arm': +- cmd += ' -target armv5te-none-linux-androideabi' +- elif arch == 'aarch64': +- cmd += ' -target aarch64-none-linux-android' +- elif arch == 'mips': +- cmd += ' -target mipsel-none-linux-android' +- cmd += ' -mips32' +- elif arch == 'mips64': +- cmd += ' -target mips64el-none-linux-android' +- elif arch == 'x86': +- cmd += ' -target i686-none-linux-android' +- elif arch == 'x86_64': +- cmd += ' -target x86_64-none-linux-android' + cmd += ' -lc -lm' + else: + cmd += " -pthread" + +- if "SYSROOT" in SDK: ++ if "SYSROOT" in SDK and GetTarget() != 'android': + cmd += " --sysroot=%s -no-canonical-prefixes" % (SDK["SYSROOT"]) + + if LDFLAGS != "": From de3ff4891eac57cc9e666b50ab2e88966d044b97 Mon Sep 17 00:00:00 2001 From: Ansh Dadwal Date: Fri, 17 May 2024 17:17:53 +0530 Subject: [PATCH 4/4] RustCompiledComponentsRecipe`: support for changing toolchain --- pythonforandroid/recipe.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pythonforandroid/recipe.py b/pythonforandroid/recipe.py index aa10434525..fcd535b0ec 100644 --- a/pythonforandroid/recipe.py +++ b/pythonforandroid/recipe.py @@ -1353,6 +1353,8 @@ class RustCompiledComponentsRecipe(PyProjectRecipe): "x86_64": "x86_64-linux-android", "x86": "i686-linux-android", } + # Rust toolchain to be used for building + toolchain = "stable" call_hostpython_via_targetpython = False @@ -1365,6 +1367,7 @@ def get_recipe_env(self, arch, **kwargs): build_target.upper().replace("-", "_") ) env["CARGO_BUILD_TARGET"] = build_target + env["TARGET"] = build_target env[cargo_linker_name] = join( self.ctx.ndk.llvm_prebuilt_dir, "bin", @@ -1386,10 +1389,6 @@ def get_recipe_env(self, arch, **kwargs): realpython_dir, "android-build", "build", "lib.linux-*-{}/".format(self.python_major_minor_version), ))[0]) - - info_main("Ensuring rust build toolchain") - shprint(sh.rustup, "target", "add", build_target) - # Add host python to PATH env["PATH"] = ("{hostpython_dir}:{old_path}").format( hostpython_dir=Recipe.get_recipe( @@ -1399,10 +1398,16 @@ def get_recipe_env(self, arch, **kwargs): ) return env + def ensure_rust_toolchain(self, arch): + info_main("Ensuring rust build toolchain : {}".format(self.toolchain)) + shprint(sh.rustup, "toolchain", "install", self.toolchain) + shprint(sh.rustup, "target", "add", "--toolchain", self.toolchain, self.RUST_ARCH_CODES[arch.arch]) + shprint(sh.rustup, "default", self.toolchain) + def check_host_deps(self): if not hasattr(sh, "rustup"): error( - "`rustup` was not found on host system." + "\n`rustup` was not found on host system." "Please install it using :" "\n`curl https://sh.rustup.rs -sSf | sh`\n" ) @@ -1410,6 +1415,7 @@ def check_host_deps(self): def build_arch(self, arch): self.check_host_deps() + self.ensure_rust_toolchain(arch) super().build_arch(arch)