From 4271ab0e0c8b4f637b35ea9598c563425f216bef Mon Sep 17 00:00:00 2001 From: Mirko Galimberti Date: Fri, 13 May 2022 22:52:54 +0200 Subject: [PATCH] Some progress on new prerequisites management --- .github/workflows/push.yml | 3 - ci/makefiles/osx.mk | 4 +- pythonforandroid/prerequisites.py | 168 +++++++++++++++++-- pythonforandroid/toolchain.py | 3 +- tests/test_prerequisites.py | 263 ++++++++++++++++++++++++++++++ tox.ini | 1 + 6 files changed, 421 insertions(+), 21 deletions(-) create mode 100644 tests/test_prerequisites.py diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index feff6bac88..5745e91693 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -122,7 +122,6 @@ jobs: run: | source ci/osx_ci.sh arm64_set_path_and_python_version 3.9.7 - brew install autoconf automake libtool openssl pkg-config make --file ci/makefiles/osx.mk - name: Build multi-arch apk Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) run: | @@ -207,7 +206,6 @@ jobs: run: | source ci/osx_ci.sh arm64_set_path_and_python_version 3.9.7 - brew install autoconf automake libtool openssl pkg-config make --file ci/makefiles/osx.mk - name: Build multi-arch aab Python 3 (armeabi-v7a, arm64-v8a, x86_64, x86) run: | @@ -293,7 +291,6 @@ jobs: run: | source ci/osx_ci.sh arm64_set_path_and_python_version 3.9.7 - brew install autoconf automake libtool openssl pkg-config make --file ci/makefiles/osx.mk - name: Rebuild updated recipes run: | diff --git a/ci/makefiles/osx.mk b/ci/makefiles/osx.mk index 9395053a6e..cb777ed1d1 100644 --- a/ci/makefiles/osx.mk +++ b/ci/makefiles/osx.mk @@ -1,4 +1,4 @@ -# installs java 1.8, android's SDK/NDK, cython and p4a +# installs Android's SDK/NDK, cython # The following variable/s can be override when running the file ANDROID_HOME ?= $(HOME)/.android @@ -10,4 +10,4 @@ upgrade_cython: install_android_ndk_sdk: mkdir -p $(ANDROID_HOME) - make -f ci/makefiles/android.mk JAVA_HOME=`/usr/libexec/java_home -v 13` + make -f ci/makefiles/android.mk diff --git a/pythonforandroid/prerequisites.py b/pythonforandroid/prerequisites.py index f9cbb2c993..a5392eeafc 100644 --- a/pythonforandroid/prerequisites.py +++ b/pythonforandroid/prerequisites.py @@ -4,20 +4,20 @@ import platform import os import subprocess +import shutil from pythonforandroid.logger import info, warning, error class Prerequisite(object): name = "Default" - mandatory = True - darwin_installer_is_supported = False - linux_installer_is_supported = False + mandatory = dict(linux=False, darwin=False) + installer_is_supported = dict(linux=False, darwin=False) def is_valid(self): if self.checker(): info(f"Prerequisite {self.name} is met") return (True, "") - elif not self.mandatory: + elif not self.mandatory[sys.platform]: warning( f"Prerequisite {self.name} is not met, but is marked as non-mandatory" ) @@ -73,10 +73,7 @@ def show_helper(self): raise Exception("Unsupported platform") def install_is_supported(self): - if sys.platform == "darwin": - return self.darwin_installer_is_supported - elif sys.platform == "linux": - return self.linux_installer_is_supported + return self.installer_is_supported[sys.platform] def linux_checker(self): raise Exception(f"Unsupported prerequisite check on linux for {self.name}") @@ -96,11 +93,42 @@ def darwin_helper(self): def linux_helper(self): info(f"No helper available for prerequisite: {self.name} on linux") + def _darwin_get_brew_formula_location_prefix(self, formula, installed=False): + opts = ["--installed"] if installed else [] + p = subprocess.Popen( + ["brew", "--prefix", formula, *opts], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + _stdout_res, _stderr_res = p.communicate() + + if p.returncode != 0: + error(_stderr_res.decode("utf-8").strip()) + return None + else: + return _stdout_res.decode("utf-8").strip() + + +class HomebrewPrerequisite(Prerequisite): + name = "homebrew" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=False) + + def darwin_checker(self): + return shutil.which("brew") is not None + + def darwin_helper(self): + info( + "Installer for homebrew is not yet supported on macOS," + "the nice news is that the installation process is easy!" + "See: https://brew.sh for further instructions." + ) + class JDKPrerequisite(Prerequisite): name = "JDK" - mandatory = True - darwin_installer_is_supported = True + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) min_supported_version = 11 def darwin_checker(self): @@ -216,13 +244,123 @@ def darwin_installer(self): os.environ["JAVA_HOME"] = jdk_path -def check_and_install_default_prerequisites(): - DEFAULT_PREREQUISITES = dict(darwin=[JDKPrerequisite()], linux=[], all_platforms=[]) +class OpenSSLPrerequisite(Prerequisite): + name = "openssl@1.1" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) + + def darwin_checker(self): + return ( + self._darwin_get_brew_formula_location_prefix("openssl@1.1", installed=True) + is not None + ) + + def darwin_installer(self): + info("Installing OpenSSL ...") + subprocess.check_output(["brew", "install", "openssl@1.1"]) + + +class AutoconfPrerequisite(Prerequisite): + name = "autoconf" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) + + def darwin_checker(self): + return ( + self._darwin_get_brew_formula_location_prefix("autoconf", installed=True) + is not None + ) + + def darwin_installer(self): + info("Installing Autoconf ...") + subprocess.check_output(["brew", "install", "autoconf"]) + + +class AutomakePrerequisite(Prerequisite): + name = "automake" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) - required_prerequisites = ( - DEFAULT_PREREQUISITES["all_platforms"] + DEFAULT_PREREQUISITES[sys.platform] + def darwin_checker(self): + return ( + self._darwin_get_brew_formula_location_prefix("automake", installed=True) + is not None + ) + + def darwin_installer(self): + info("Installing Automake ...") + subprocess.check_output(["brew", "install", "automake"]) + + +class LibtoolPrerequisite(Prerequisite): + name = "libtool" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) + + def darwin_checker(self): + return ( + self._darwin_get_brew_formula_location_prefix("libtool", installed=True) + is not None + ) + + def darwin_installer(self): + info("Installing Libtool ...") + subprocess.check_output(["brew", "install", "libtool"]) + + +class PkgConfigPrerequisite(Prerequisite): + name = "pkg-config" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) + + def darwin_checker(self): + return ( + self._darwin_get_brew_formula_location_prefix("pkg-config", installed=True) + is not None + ) + + def darwin_installer(self): + info("Installing Pkg-Config ...") + subprocess.check_output(["brew", "install", "pkg-config"]) + + +class CmakePrerequisite(Prerequisite): + name = "cmake" + mandatory = dict(linux=False, darwin=True) + installer_is_supported = dict(linux=False, darwin=True) + + def darwin_checker(self): + return ( + self._darwin_get_brew_formula_location_prefix("cmake", installed=True) + is not None + ) + + def darwin_installer(self): + info("Installing cmake ...") + subprocess.check_output(["brew", "install", "cmake"]) + + +def get_required_prerequisites(platform="linux"): + DEFAULT_PREREQUISITES = dict( + darwin=[ + HomebrewPrerequisite(), + AutoconfPrerequisite(), + AutomakePrerequisite(), + LibtoolPrerequisite(), + PkgConfigPrerequisite(), + CmakePrerequisite(), + OpenSSLPrerequisite(), + JDKPrerequisite(), + ], + linux=[], + all_platforms=[], ) + return DEFAULT_PREREQUISITES["all_platforms"] + DEFAULT_PREREQUISITES[platform] + + +def check_and_install_default_prerequisites(): + prerequisites_not_met = [] warning( @@ -232,7 +370,7 @@ def check_and_install_default_prerequisites(): # Phase 1: Check if all prerequisites are met and add the ones # which are not to `prerequisites_not_met` - for prerequisite in required_prerequisites: + for prerequisite in get_required_prerequisites(sys.platform): if not prerequisite.is_valid(): prerequisites_not_met.append(prerequisite) diff --git a/pythonforandroid/toolchain.py b/pythonforandroid/toolchain.py index 520012d55b..9badb0d93f 100644 --- a/pythonforandroid/toolchain.py +++ b/pythonforandroid/toolchain.py @@ -67,7 +67,8 @@ def check_python_dependencies(): exit(1) -check_and_install_default_prerequisites() +if not environ.get('SKIP_PREREQUISITES_CHECK', '0') == '1': + check_and_install_default_prerequisites() check_python_dependencies() diff --git a/tests/test_prerequisites.py b/tests/test_prerequisites.py new file mode 100644 index 0000000000..7c8ce9f5ff --- /dev/null +++ b/tests/test_prerequisites.py @@ -0,0 +1,263 @@ +import unittest +from unittest import mock + +from pythonforandroid.prerequisites import ( + JDKPrerequisite, + HomebrewPrerequisite, + OpenSSLPrerequisite, + AutoconfPrerequisite, + AutomakePrerequisite, + LibtoolPrerequisite, + PkgConfigPrerequisite, + CmakePrerequisite, + get_required_prerequisites, +) + + +class PrerequisiteSetUpBaseClass: + def setUp(self): + self.mandatory = dict(linux=False, darwin=False) + self.installer_is_supported = dict(linux=False, darwin=False) + + def test_is_mandatory_on_darwin(self): + assert self.prerequisite.mandatory["darwin"] == self.mandatory["darwin"] + + def test_is_mandatory_on_linux(self): + assert self.prerequisite.mandatory["linux"] == self.mandatory["linux"] + + def test_installer_is_supported_on_darwin(self): + assert ( + self.prerequisite.installer_is_supported["darwin"] + == self.installer_is_supported["darwin"] + ) + + def test_installer_is_supported_on_linux(self): + assert ( + self.prerequisite.installer_is_supported["linux"] + == self.installer_is_supported["linux"] + ) + + +class TestJDKPrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = JDKPrerequisite() + + +class TestBrewPrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=False) + self.prerequisite = HomebrewPrerequisite() + + @mock.patch("shutil.which") + def test_darwin_checker(self, shutil_which): + shutil_which.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + shutil_which.return_value = "/opt/homebrew/bin/brew" + self.assertTrue(self.prerequisite.darwin_checker()) + + @mock.patch("pythonforandroid.prerequisites.info") + def test_darwin_helper(self, info): + self.prerequisite.darwin_helper() + info.assert_called_once_with( + "Installer for homebrew is not yet supported on macOS," + "the nice news is that the installation process is easy!" + "See: https://brew.sh for further instructions." + ) + + +class TestOpenSSLPrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = OpenSSLPrerequisite() + + @mock.patch( + "pythonforandroid.prerequisites.Prerequisite._darwin_get_brew_formula_location_prefix" + ) + def test_darwin_checker(self, _darwin_get_brew_formula_location_prefix): + _darwin_get_brew_formula_location_prefix.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.return_value = ( + "/opt/homebrew/opt/openssl@1.1" + ) + self.assertTrue(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.assert_called_with( + "openssl@1.1", installed=True + ) + + @mock.patch("pythonforandroid.prerequisites.subprocess.check_output") + def test_darwin_installer(self, check_output): + self.prerequisite.darwin_installer() + check_output.assert_called_once_with(["brew", "install", "openssl@1.1"]) + + +class TestAutoconfPrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = AutoconfPrerequisite() + + @mock.patch( + "pythonforandroid.prerequisites.Prerequisite._darwin_get_brew_formula_location_prefix" + ) + def test_darwin_checker(self, _darwin_get_brew_formula_location_prefix): + _darwin_get_brew_formula_location_prefix.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.return_value = ( + "/opt/homebrew/opt/autoconf" + ) + self.assertTrue(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.assert_called_with( + "autoconf", installed=True + ) + + @mock.patch("pythonforandroid.prerequisites.subprocess.check_output") + def test_darwin_installer(self, check_output): + self.prerequisite.darwin_installer() + check_output.assert_called_once_with(["brew", "install", "autoconf"]) + + +class TestAutomakePrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = AutomakePrerequisite() + + @mock.patch( + "pythonforandroid.prerequisites.Prerequisite._darwin_get_brew_formula_location_prefix" + ) + def test_darwin_checker(self, _darwin_get_brew_formula_location_prefix): + _darwin_get_brew_formula_location_prefix.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.return_value = ( + "/opt/homebrew/opt/automake" + ) + self.assertTrue(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.assert_called_with( + "automake", installed=True + ) + + @mock.patch("pythonforandroid.prerequisites.subprocess.check_output") + def test_darwin_installer(self, check_output): + self.prerequisite.darwin_installer() + check_output.assert_called_once_with(["brew", "install", "automake"]) + + +class TestLibtoolPrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = LibtoolPrerequisite() + + @mock.patch( + "pythonforandroid.prerequisites.Prerequisite._darwin_get_brew_formula_location_prefix" + ) + def test_darwin_checker(self, _darwin_get_brew_formula_location_prefix): + _darwin_get_brew_formula_location_prefix.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.return_value = ( + "/opt/homebrew/opt/libtool" + ) + self.assertTrue(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.assert_called_with( + "libtool", installed=True + ) + + @mock.patch("pythonforandroid.prerequisites.subprocess.check_output") + def test_darwin_installer(self, check_output): + self.prerequisite.darwin_installer() + check_output.assert_called_once_with(["brew", "install", "libtool"]) + + +class TestPkgConfigPrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = PkgConfigPrerequisite() + + @mock.patch( + "pythonforandroid.prerequisites.Prerequisite._darwin_get_brew_formula_location_prefix" + ) + def test_darwin_checker(self, _darwin_get_brew_formula_location_prefix): + _darwin_get_brew_formula_location_prefix.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.return_value = ( + "/opt/homebrew/opt/pkg-config" + ) + self.assertTrue(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.assert_called_with( + "pkg-config", installed=True + ) + + @mock.patch("pythonforandroid.prerequisites.subprocess.check_output") + def test_darwin_installer(self, check_output): + self.prerequisite.darwin_installer() + check_output.assert_called_once_with(["brew", "install", "pkg-config"]) + + +class TestCmakePrerequisite(PrerequisiteSetUpBaseClass, unittest.TestCase): + def setUp(self): + super().setUp() + self.mandatory = dict(linux=False, darwin=True) + self.installer_is_supported = dict(linux=False, darwin=True) + self.prerequisite = CmakePrerequisite() + + @mock.patch( + "pythonforandroid.prerequisites.Prerequisite._darwin_get_brew_formula_location_prefix" + ) + def test_darwin_checker(self, _darwin_get_brew_formula_location_prefix): + _darwin_get_brew_formula_location_prefix.return_value = None + self.assertFalse(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.return_value = ( + "/opt/homebrew/opt/cmake" + ) + self.assertTrue(self.prerequisite.darwin_checker()) + _darwin_get_brew_formula_location_prefix.assert_called_with( + "cmake", installed=True + ) + + @mock.patch("pythonforandroid.prerequisites.subprocess.check_output") + def test_darwin_installer(self, check_output): + self.prerequisite.darwin_installer() + check_output.assert_called_once_with(["brew", "install", "cmake"]) + + +class TestDefaultPrerequisitesCheckandInstall(unittest.TestCase): + + def test_default_darwin_prerequisites_set(self): + self.assertListEqual( + [ + p.__class__.__name__ + for p in get_required_prerequisites(platform="darwin") + ], + [ + "HomebrewPrerequisite", + "AutoconfPrerequisite", + "AutomakePrerequisite", + "LibtoolPrerequisite", + "PkgConfigPrerequisite", + "CmakePrerequisite", + "OpenSSLPrerequisite", + "JDKPrerequisite", + ], + ) + + def test_default_linux_prerequisites_set(self): + self.assertListEqual( + [ + p.__class__.__name__ + for p in get_required_prerequisites(platform="linux") + ], + [ + ], + ) diff --git a/tox.ini b/tox.ini index 223ead06b3..ceafe446b5 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,7 @@ commands = pytest {posargs:tests/} passenv = GITHUB_* setenv = PYTHONPATH={toxinidir} + SKIP_PREREQUISITES_CHECK=1 [testenv:py3] # for py3 env we will get code coverage