Skip to content

Add support for Python 3.11 and make it the default while building hostpython3 and python3 #2850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Nov 18, 2023
Merged

Add support for Python 3.11 and make it the default while building hostpython3 and python3 #2850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
78db832
python: update to 3.11.5
T-Dynamos Jul 8, 2023
1efdad3
scipy: update to 1.11.3
T-Dynamos Oct 1, 2023
d2be9e1
add `pybind11` to `ci/constants.py` in BROKEN_RECIPES_PYTHON3
T-Dynamos Nov 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion ci/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,8 @@ class TargetPython(Enum):
'boost',
# libtorrent gives errors (requires boost. Also, see issue #2809, to start with)
'libtorrent',

# pybind11 build fails on macos
'pybind11',
])

BROKEN_RECIPES = {
Expand Down
3 changes: 2 additions & 1 deletion pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ protected static ArrayList<String> getLibraries(File libsDir) {
libsList.add("python3.8");
libsList.add("python3.9");
libsList.add("python3.10");
libsList.add("python3.11");
libsList.add("main");
return libsList;
}
Expand All @@ -74,7 +75,7 @@ public static void loadLibraries(File filesDir, File libsDir) {
// load, and it has failed, give a more
// general error
Log.v(TAG, "Library loading error: " + e.getMessage());
if (lib.startsWith("python3.10") && !foundPython) {
if (lib.startsWith("python3.11") && !foundPython) {
throw new RuntimeException("Could not load any libpythonXXX.so");
} else if (lib.startsWith("python")) {
continue;
Expand Down
2 changes: 1 addition & 1 deletion pythonforandroid/recipes/cython/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

class CythonRecipe(CompiledComponentsPythonRecipe):

version = '0.29.28'
version = '0.29.36'
url = 'https://github.com/cython/cython/archive/{version}.tar.gz'
site_packages_name = 'cython'
depends = ['setuptools']
Expand Down
2 changes: 1 addition & 1 deletion pythonforandroid/recipes/hostpython3/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class HostPython3Recipe(Recipe):
:class:`~pythonforandroid.python.HostPythonRecipe`
'''

version = '3.10.10'
version = '3.11.5'
name = 'hostpython3'

build_subdir = 'native-build'
Expand Down
2 changes: 1 addition & 1 deletion pythonforandroid/recipes/pybind11/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

class Pybind11Recipe(PythonRecipe):

version = '2.9.0'
version = '2.11.1'
url = 'https://github.com/pybind/pybind11/archive/refs/tags/v{version}.zip'
depends = ['setuptools']
call_hostpython_via_targetpython = False
Expand Down
27 changes: 20 additions & 7 deletions pythonforandroid/recipes/python3/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import sh
import subprocess

from multiprocessing import cpu_count
from os import environ, utime
from os.path import dirname, exists, join
from pathlib import Path
Expand Down Expand Up @@ -56,7 +55,7 @@ class Python3Recipe(TargetPythonRecipe):
:class:`~pythonforandroid.python.GuestPythonRecipe`
'''

version = '3.10.10'
version = '3.11.5'
url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'
name = 'python3'

Expand All @@ -71,15 +70,17 @@ class Python3Recipe(TargetPythonRecipe):
# Python 3.8.1 & 3.9.X
('patches/py3.8.1.patch', version_starts_with("3.8")),
('patches/py3.8.1.patch', version_starts_with("3.9")),
('patches/py3.8.1.patch', version_starts_with("3.10"))
('patches/py3.8.1.patch', version_starts_with("3.10")),
('patches/cpython-311-ctypes-find-library.patch', version_starts_with("3.11")),
]

if shutil.which('lld') is not None:
patches = patches + [
patches += [
("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")),
("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")),
("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")),
("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10"))
("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10")),
("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.11")),
]

depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
Expand All @@ -101,7 +102,12 @@ class Python3Recipe(TargetPythonRecipe):
'ac_cv_header_sys_eventfd_h=no',
'--prefix={prefix}',
'--exec-prefix={exec_prefix}',
'--enable-loadable-sqlite-extensions')
'--enable-loadable-sqlite-extensions'
)

if version_starts_with("3.11"):
configure_args += ('--with-build-python={python_host_bin}',)

'''The configure arguments needed to build the python recipe. Those are
used in method :meth:`build_arch` (if not overwritten like python3's
recipe does).
Expand Down Expand Up @@ -323,12 +329,19 @@ def build_arch(self, arch):
*(' '.join(self.configure_args).format(
android_host=env['HOSTARCH'],
android_build=android_build,
python_host_bin=join(self.get_recipe(
'host' + self.name, self.ctx
).get_path_to_python(), "python3"),
prefix=sys_prefix,
exec_prefix=sys_exec_prefix)).split(' '),
_env=env)

# Python build does not seem to play well with make -j option from Python 3.11 and onwards
# Before losing some time, please check issue
# https://github.com/python/cpython/issues/101295 , as the root cause looks similar
shprint(
sh.make, 'all', '-j', str(cpu_count()),
sh.make,
'all',
'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
_env=env
)
Expand Down
19 changes: 19 additions & 0 deletions pythonforandroid/recipes/python3/patches/cpython-311-ctypes-find-library.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
--- Python-3.11.5/Lib/ctypes/util.py 2023-08-24 17:39:18.000000000 +0530
+++ Python-3.11.5.mod/Lib/ctypes/util.py 2023-11-18 22:12:17.356160615 +0530
@@ -4,7 +4,15 @@
import sys

# find_library(name) returns the pathname of a library, or None.
-if os.name == "nt":
+
+# This patch overrides the find_library to look in the right places on
+# Android
+if True:
+ from android._ctypes_library_finder import find_library as _find_lib
+ def find_library(name):
+ return _find_lib(name)
+
+elif os.name == "nt":

def _get_build_version():
"""Return the version of MSVC that was used to build Python.
18 changes: 14 additions & 4 deletions pythonforandroid/recipes/scipy/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
from multiprocessing import cpu_count
from os.path import join
from os import environ
from pythonforandroid.util import build_platform
import sh
from pythonforandroid.logger import shprint
from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
from pythonforandroid.util import build_platform, current_directory


def arch_to_toolchain(arch):
Expand All @@ -13,12 +15,14 @@ def arch_to_toolchain(arch):

class ScipyRecipe(CompiledComponentsPythonRecipe):

version = '1.8.1'
url = f'https://github.com/scipy/scipy/releases/download/v{version}/scipy-{version}.zip'
version = 'maintenance/1.11.x'
url = 'git+https://github.com/scipy/scipy.git'
git_commit = 'b430bf54b5064465983813e2cfef3fcb86c3df07' # version 1.11.3
site_packages_name = 'scipy'
depends = ['setuptools', 'cython', 'numpy', 'lapack', 'pybind11']
call_hostpython_via_targetpython = False
need_stl_shared = True
patches = ["setup.py.patch"]

def build_compiled_components(self, arch):
self.setup_extra_args = ['-j', str(cpu_count())]
Expand All @@ -30,6 +34,12 @@ def rebuild_compiled_components(self, arch, env):
super().rebuild_compiled_components(arch, env)
self.setup_extra_args = []

def download_file(self, url, target, cwd=None):
super().download_file(url, target, cwd=cwd)
with current_directory(target):
shprint(sh.git, 'fetch', '--unshallow')
shprint(sh.git, 'checkout', self.git_commit)

def get_recipe_env(self, arch):
env = super().get_recipe_env(arch)
arch_env = arch.get_env()
Expand Down
Loading