From f9bf0250a5724adeba62074d17747b5dfa0e255b Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Thu, 14 Jan 2021 23:43:03 +0300 Subject: [PATCH 1/7] Restore set +e; dropped with `before_cache` step --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3c2efde8..1735d655 100644 --- a/.travis.yml +++ b/.travis.yml @@ -922,7 +922,7 @@ script: | after_success: | # Upload wheels to pypi if tag is set - set -x + set -x; set +e if [ -n "$TRAVIS_TAG" ]; then From 964ef123b5ac077723b6cfef387d0b9b14d3a467 Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Tue, 8 Dec 2020 23:17:28 +0300 Subject: [PATCH 2/7] Use MacOS 10.15.5 for build To fix warnings like this: Warning: You are using macOS 10.13. 5We (and Apple) do not provide support for this old version. You will encounter build failures with some formulae. Also use Qt-5.15.2 which is installed in MacOS 10.15.5 --- .travis.yml | 32 ++++++++++++++++---------------- travis_config.sh | 11 ++--------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1735d655..9b3c7804 100644 --- a/.travis.yml +++ b/.travis.yml @@ -79,7 +79,7 @@ jobs: - stage: final - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 @@ -88,7 +88,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 @@ -97,7 +97,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.8 - ENABLE_CONTRIB=0 @@ -106,7 +106,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.9 - ENABLE_CONTRIB=0 @@ -117,7 +117,7 @@ jobs: # headless builds for MacOS - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=0 @@ -126,7 +126,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=0 @@ -135,7 +135,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.8 - ENABLE_CONTRIB=0 @@ -144,7 +144,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.9 - ENABLE_CONTRIB=0 @@ -155,7 +155,7 @@ jobs: # Contrib builds for MacOS - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 @@ -164,7 +164,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 @@ -173,7 +173,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.8 - ENABLE_CONTRIB=1 @@ -182,7 +182,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.9 - ENABLE_CONTRIB=1 @@ -193,7 +193,7 @@ jobs: # headless contrib builds for MacOS - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.6 - ENABLE_CONTRIB=1 @@ -202,7 +202,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.7 - ENABLE_CONTRIB=1 @@ -211,7 +211,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.8 - ENABLE_CONTRIB=1 @@ -220,7 +220,7 @@ jobs: - os: osx language: generic - osx_image: xcode9.4 + osx_image: xcode11.6 env: - MB_PYTHON_VERSION=3.9 - ENABLE_CONTRIB=1 diff --git a/travis_config.sh b/travis_config.sh index 9ba0bb11..7a358ec0 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -115,15 +115,8 @@ function pre_build { brew install ffmpeg_opencv fi - # echo 'Installing qt5' - - # if [ -n "$CACHE_STAGE" ]; then - # echo "Qt5 has bottle, no caching needed" - # else - # brew switch qt 5.13.2 - # brew pin qt - # export PATH="/usr/local/opt/qt/bin:$PATH" - # fi + echo 'Set up qt5' + export PATH="/usr/local/opt/qt/bin:$PATH" if [ -n "$CACHE_STAGE" ]; then brew_go_bootstrap_mode 0 From 673effc2828070e84d4a5a4658184facc7b4fa75 Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Fri, 22 Jan 2021 13:20:41 +0300 Subject: [PATCH 3/7] Update multibuild submodule to the latest version --- multibuild | 2 +- travis_config.sh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/multibuild b/multibuild index 8882150d..94f249a0 160000 --- a/multibuild +++ b/multibuild @@ -1 +1 @@ -Subproject commit 8882150df6529658700b66bec124dfb77eefca26 +Subproject commit 94f249a0127c81d563f74bb71440a0f6747a67dd diff --git a/travis_config.sh b/travis_config.sh index 7a358ec0..4c7f0156 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -16,7 +16,7 @@ function bdist_wheel_cmd { local abs_wheelhouse=$1 CI_BUILD=1 pip wheel --verbose --wheel-dir="$PWD/dist" . $BDIST_PARAMS cp dist/*.whl $abs_wheelhouse - if [ -z "$IS_OSX" ]; then + if [ -z "$IS_MACOS" ]; then TOOLS_PATH=/opt/_internal/tools /opt/python/cp37-cp37m/bin/python -m venv $TOOLS_PATH source $TOOLS_PATH/bin/activate @@ -26,7 +26,7 @@ function bdist_wheel_cmd { if [ -n "$USE_CCACHE" -a -z "$BREW_BOOTSTRAP_MODE" ]; then ccache -s; fi } -if [ -n "$IS_OSX" ]; then +if [ -n "$IS_MACOS" ]; then echo " > OSX environment " export MAKEFLAGS="-j$(sysctl -n hw.ncpu)" else @@ -35,7 +35,7 @@ else export MAKEFLAGS="-j$(grep -E '^processor[[:space:]]*:' /proc/cpuinfo | wc -l)" fi -if [ -n "$IS_OSX" ]; then +if [ -n "$IS_MACOS" ]; then source travis_osx_brew_cache.sh @@ -92,7 +92,7 @@ function pre_build { echo "Starting pre-build" set -e -o pipefail - if [ -n "$IS_OSX" ]; then + if [ -n "$IS_MACOS" ]; then echo "Running for OSX" local CACHE_STAGE; (echo "$TRAVIS_BUILD_STAGE_NAME" | grep -qiF "final") || CACHE_STAGE=1 @@ -136,7 +136,7 @@ function run_tests { echo "Run tests..." echo $PWD - if [ -n "$IS_OSX" ]; then + if [ -n "$IS_MACOS" ]; then echo "Running for OS X" cd ../tests/ else From 3826d255472073dff9b4e9879255ecd948d61038 Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Wed, 30 Dec 2020 23:30:45 +0300 Subject: [PATCH 4/7] Rework Mac OS build Use cmake && make to prebuild opencv libraries and use pip to pack them --- .travis.yml | 35 +++++++++ setup-osx.py | 138 +++++++++++++++++++++++++++++++++ tools/python_path.py | 27 +++++++ travis_config.sh | 7 +- travis_osx_build.sh | 178 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 384 insertions(+), 1 deletion(-) create mode 100644 setup-osx.py create mode 100644 tools/python_path.py create mode 100644 travis_osx_build.sh diff --git a/.travis.yml b/.travis.yml index 9b3c7804..fa101424 100644 --- a/.travis.yml +++ b/.travis.yml @@ -85,6 +85,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.13.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -94,6 +96,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.14.5 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -103,6 +107,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.17.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -112,6 +118,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.19.3 + cache: + directories: $HOME/.ccache # headless builds for MacOS @@ -123,6 +131,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.13.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -132,6 +142,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.14.5 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -141,6 +153,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.17.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -150,6 +164,8 @@ jobs: - ENABLE_CONTRIB=0 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.19.3 + cache: + directories: $HOME/.ccache # Contrib builds for MacOS @@ -161,6 +177,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.13.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -170,6 +188,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.14.5 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -179,6 +199,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.17.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -188,6 +210,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=0 - TEST_DEPENDS=numpy==1.19.3 + cache: + directories: $HOME/.ccache # headless contrib builds for MacOS @@ -199,6 +223,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.13.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -208,6 +234,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.14.5 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -217,6 +245,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.17.3 + cache: + directories: $HOME/.ccache - os: osx language: generic @@ -226,6 +256,8 @@ jobs: - ENABLE_CONTRIB=1 - ENABLE_HEADLESS=1 - TEST_DEPENDS=numpy==1.19.3 + cache: + directories: $HOME/.ccache # default builds for Linux @@ -896,6 +928,9 @@ install: | if [[ $SDIST == 1 ]]; then python -m pip install --upgrade pip python -m pip install scikit-build + git submodule sync + git submodule update --init --recursive opencv + git submodule update --init --recursive opencv_contrib python setup.py sdist else build_wheel $REPO_DIR $PLAT diff --git a/setup-osx.py b/setup-osx.py new file mode 100644 index 00000000..5af8e429 --- /dev/null +++ b/setup-osx.py @@ -0,0 +1,138 @@ +import io +import os +import os.path +import sys +import runpy +import re +import sysconfig +import setuptools + + +def main(): + os.chdir(os.path.dirname(os.path.abspath(__file__))) + + minimum_supported_numpy = "1.13.3" + if sys.version_info[:2] >= (3, 6): + minimum_supported_numpy = "1.13.3" + if sys.version_info[:2] >= (3, 7): + minimum_supported_numpy = "1.14.5" + if sys.version_info[:2] >= (3, 8): + minimum_supported_numpy = "1.17.3" + if sys.version_info[:2] >= (3, 9): + minimum_supported_numpy = "1.19.3" + + numpy_version = "numpy>=%s" % minimum_supported_numpy + + build_contrib = get_build_env_var_by_name("contrib") + build_headless = get_build_env_var_by_name("headless") + build_java = "ON" if get_build_env_var_by_name("java") else "OFF" + + version = {} + here = os.path.abspath(os.path.dirname(__file__)) + version_file = os.path.join(here, "cv2", "version.py") + + # generate a fresh version.py always when Git repository exists + if os.path.exists(".git"): + old_args = sys.argv.copy() + sys.argv = ["", str(build_contrib), str(build_headless), str(False)] + runpy.run_path("find_version.py", run_name="__main__") + sys.argv = old_args + + with open(version_file) as fp: + exec(fp.read(), version) + + package_version = version["opencv_version"] + build_contrib = version["contrib"] + build_headless = version["headless"] + + package_name = "opencv-python" + + if build_contrib and not build_headless: + package_name = "opencv-contrib-python" + + if build_contrib and build_headless: + package_name = "opencv-contrib-python-headless" + + if build_headless and not build_contrib: + package_name = "opencv-python-headless" + + long_description = io.open("README.md", encoding="utf-8").read() + + packages = ["cv2", "cv2.data"] + + package_data = { + "cv2": ["*%s" % sysconfig.get_config_vars().get("SO"), "version.py"] + + (["*.dll"] if os.name == "nt" else []) + + ["LICENSE.txt", "LICENSE-3RD-PARTY.txt"], + "cv2.data": ["*.xml"], + } + + setuptools.setup( + name=package_name, + version=package_version, + url="https://github.com/skvark/opencv-python", + license="MIT", + description="Wrapper package for OpenCV python bindings.", + long_description=long_description, + long_description_content_type="text/markdown", + packages=packages, + package_data=package_data, + include_package_data=True, + maintainer="Olli-Pekka Heinisuo", + ext_modules=EmptyListWithLength(), + install_requires=numpy_version, + python_requires=">=3.6", + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: Education", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: MacOS", + "Operating System :: Microsoft :: Windows", + "Operating System :: POSIX", + "Operating System :: Unix", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: C++", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Image Recognition", + "Topic :: Software Development", + ], + ) + + +def get_build_env_var_by_name(flag_name): + flag_set = False + + try: + flag_set = bool(int(os.getenv("ENABLE_" + flag_name.upper(), None))) + except Exception: + pass + + if not flag_set: + try: + flag_set = bool(int(open(flag_name + ".enabled").read(1))) + except Exception: + pass + + return flag_set + + +# This creates a list which is empty but returns a length of 1. +# Should make the wheel a binary distribution and platlib compliant. +class EmptyListWithLength(list): + def __len__(self): + return 1 + + +if __name__ == "__main__": + main() diff --git a/tools/python_path.py b/tools/python_path.py new file mode 100644 index 00000000..5d19e72e --- /dev/null +++ b/tools/python_path.py @@ -0,0 +1,27 @@ +import sys + + +def main(): + if len(sys.argv) < 2: + print("Usage: ", sys.argv[0], "bin|lib|include") + return 1 + + if (sys.argv[1] == "bin"): + print(sys.executable) + elif (sys.argv[1] == "lib"): + from skbuild import cmaker + python_version = cmaker.CMaker.get_python_version() + python_lib_path = cmaker.CMaker.get_python_library(python_version).replace("\\", "/") + # FIXME: Wrong extension: + # import os + # from sysconfig import get_config_var + # python_lib_path = os.path.join(get_config_var('LIBDIR'), get_config_var('LIBRARY')) + print(python_lib_path) + else: + from sysconfig import get_paths + info = get_paths() + print(info['platinclude']) + + +if __name__ == "__main__": + main() diff --git a/travis_config.sh b/travis_config.sh index 4c7f0156..675d0cf2 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -14,7 +14,12 @@ function bdist_wheel_cmd { # copied from multibuild's common_utils.sh # add osx deployment target so it doesnt default to 10.6 local abs_wheelhouse=$1 - CI_BUILD=1 pip wheel --verbose --wheel-dir="$PWD/dist" . $BDIST_PARAMS + if [ -n "$IS_MACOS" ]; then + source travis_osx_build.sh + build_bdist_osx_wheel $@ + else + CI_BUILD=1 pip wheel --verbose --wheel-dir="$PWD/dist" . $BDIST_PARAMS + fi cp dist/*.whl $abs_wheelhouse if [ -z "$IS_MACOS" ]; then TOOLS_PATH=/opt/_internal/tools diff --git a/travis_osx_build.sh b/travis_osx_build.sh new file mode 100644 index 00000000..33c44793 --- /dev/null +++ b/travis_osx_build.sh @@ -0,0 +1,178 @@ +function package_name() { + local name="opencv" + if [ $ENABLE_CONTRIB -ne 0 ]; then + name+="_contrib" + fi + name+="_python" + if [ $ENABLE_HEADLESS -ne 0 ]; then + name+="_headless" + fi + wheel_name=$name +} + +function pre_build_osx { + local repo_dir=$(abspath ${1:-$REPO_DIR}) + local build_dir="$repo_dir/opencv/build" + local num_cpus=$(sysctl -n hw.ncpu) + num_cpus=${num_cpus:-4} + + cd "$repo_dir" + git submodule sync + git submodule update --init --recursive opencv + git submodule update --init --recursive opencv_contrib + + pip install scikit-build + pip install numpy + + if [ ! -d "$build_dir" ]; then + mkdir "$build_dir" + fi + + local CMAKE_OPTS=( + -G "Unix Makefiles" + -DPYTHON3_EXECUTABLE="$(python "$repo_dir/tools/python_path.py" bin)" + -DPYTHON3_INCLUDE_DIR="$(python "$repo_dir/tools/python_path.py" include)" + -DPYTHON3_LIBRARY="$(python "$repo_dir/tools/python_path.py" lib)" + -DBUILD_opencv_python3=ON + -DBUILD_opencv_python2=OFF + -DBUILD_opencv_java=OFF + -DOPENCV_SKIP_PYTHON_LOADER=ON + -DOPENCV_PYTHON3_INSTALL_PATH=python + -DINSTALL_CREATE_DISTRIB=ON + -DBUILD_opencv_apps=OFF + -DBUILD_SHARED_LIBS=OFF + -DBUILD_TESTS=OFF + -DBUILD_PERF_TESTS=OFF + -DBUILD_DOCS=OFF + ) + if [ $ENABLE_CONTRIB -ne 0 ]; then + CMAKE_OPTS+=(-DOPENCV_EXTRA_MODULES_PATH="$repo_dir/opencv_contrib/modules") + fi + if [ $ENABLE_HEADLESS -eq 0 ]; then + export PKG_CONFIG_PATH="/usr/local/opt/qt/lib/pkgconfig":$PKG_CONFIG_PATH + export CMAKE_PREFIX_PATH="/usr/local/Cellar/qt/5.15.1" + CMAKE_OPTS+=(-DWITH_QT=5) + else + CMAKE_OPTS+=( + -DWITH_WIN32UI=OFF + -DWITH_QT=OFF + -DWITH_GTK=OFF + # -DWITH_MSMF=OFF + ) + fi + + # Clear ccache stats + ccache -z + + # Configure build + cd "$build_dir" + cmake "${CMAKE_OPTS[@]}" .. + # $ pkg-config --libs opencv4 | sed -e 's| |\n|g' | sed -e 's|^-l||g' | tac + CV_MODULES=( + opencv_core + opencv_imgproc + opencv_photo + opencv_xphoto + opencv_flann + opencv_features2d + opencv_imgcodecs + opencv_calib3d + opencv_objdetect + opencv_xobjdetect + opencv_video + opencv_ximgproc + opencv_ml + opencv_shape + opencv_xfeatures2d + opencv_viz + opencv_videoio + opencv_videostab + opencv_plot + opencv_dnn + opencv_text + opencv_datasets + opencv_tracking + opencv_surface_matching + opencv_optflow + opencv_superres + opencv_phase_unwrapping + opencv_structured_light + opencv_stereo + opencv_saliency + opencv_rgbd + opencv_reg + opencv_rapid + opencv_quality + opencv_mcc + opencv_line_descriptor + opencv_intensity_transform + opencv_img_hash + opencv_hfs + opencv_hdf + opencv_fuzzy + opencv_freetype + opencv_face + opencv_highgui + opencv_dpm + opencv_dnn_superres + opencv_dnn_objdetect + opencv_cvv + opencv_ccalib + opencv_bioinspired + opencv_bgsegm + opencv_aruco + opencv_alphamat + opencv_stitching + opencv_gapi + ) + for m in "${CV_MODULES[@]}"; do + if make help | grep -w "$m"; then + make -j${num_cpus} "$m" + fi + done + make -j${num_cpus} + + # Print ccache stats + ccache -s +} + +function build_osx { + local repo_dir=$(abspath ${1:-$REPO_DIR}) + local build_dir="$repo_dir/opencv/build" + package_name + + # Copy compiled python module to package + cd "$build_dir" + cp -f lib/python3/cv2*.so "$repo_dir/cv2" + + if [ $ENABLE_HEADLESS -eq 0 ]; then + if [ ! -d "$repo_dir/cv2/qt/plugins/platforms" ]; then + mkdir -p "$repo_dir/cv2/qt/plugins/platforms" + fi + cp /usr/local/Cellar/qt/5.15.1/plugins/platforms/libqcocoa.dylib "$repo_dir/cv2/qt/plugins/platforms" + fi + + cp -f "$repo_dir/opencv/data/haarcascades/"*.xml "$repo_dir/cv2/data" + # Copy licenses + cp -f "$repo_dir/LICENSE.txt" "$repo_dir/LICENSE-3RD-PARTY.txt" "$repo_dir/cv2" + + cd "$repo_dir" + cat setup-osx.py > setup.py + export ENABLE_CONTRIB + export ENABLE_HEADLESS + export ENABLE_JAVA + pip wheel --verbose --wheel-dir="$PWD/dist" . $BDIST_PARAMS + + echo 'Built wheels:' + ls -lh "$PWD/dist/${wheel_name}"*.whl +} + +function build_bdist_osx_wheel { + local repo_dir=$(abspath ${1:-$REPO_DIR}) + [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 + pre_build_osx "$repo_dir" + if [ -n "$BUILD_DEPENDS" ]; then + pip install $(pip_opts) $BUILD_DEPENDS + fi + build_osx "$repo_dir" +} From 6b9eb0f8d0e82a906032384632eb3ed68ed7f0f0 Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Wed, 20 Jan 2021 02:29:05 +0300 Subject: [PATCH 5/7] Implement build timeouts --- .travis.yml | 1 + travis_config.sh | 14 ++++++------ travis_osx_build.sh | 56 ++++++++++++++++++++++++++++++++++++++------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index fa101424..fd4d4170 100644 --- a/.travis.yml +++ b/.travis.yml @@ -934,6 +934,7 @@ install: | python setup.py sdist else build_wheel $REPO_DIR $PLAT + ret_code=$? fi set +x diff --git a/travis_config.sh b/travis_config.sh index 675d0cf2..04830721 100644 --- a/travis_config.sh +++ b/travis_config.sh @@ -7,19 +7,19 @@ echo "=== Loading config.sh === " # To see build progress function build_wheel { - build_bdist_wheel $@ + if [ -n "$IS_OSX" ]; then + source travis_osx_build.sh + build_bdist_osx_wheel $@ || return $? + else + build_bdist_wheel $@ + fi } function bdist_wheel_cmd { # copied from multibuild's common_utils.sh # add osx deployment target so it doesnt default to 10.6 local abs_wheelhouse=$1 - if [ -n "$IS_MACOS" ]; then - source travis_osx_build.sh - build_bdist_osx_wheel $@ - else - CI_BUILD=1 pip wheel --verbose --wheel-dir="$PWD/dist" . $BDIST_PARAMS - fi + CI_BUILD=1 pip wheel --verbose --wheel-dir="$PWD/dist" . $BDIST_PARAMS cp dist/*.whl $abs_wheelhouse if [ -z "$IS_MACOS" ]; then TOOLS_PATH=/opt/_internal/tools diff --git a/travis_osx_build.sh b/travis_osx_build.sh index 33c44793..f09e1b37 100644 --- a/travis_osx_build.sh +++ b/travis_osx_build.sh @@ -10,20 +10,47 @@ function package_name() { wheel_name=$name } +# Terminate the build but ensure saving the cache +function goto_exit { + local EXIT_CODE=${1:-1} + + echo "Exiting build" + + # Can't just `exit` because that would terminate the build without saving the cache + # Have to replace further actions with no-ops + + local MESSAGE=""; if [ "$EXIT_CODE" -ne 0 ]; then + MESSAGE='Building package took too long. Restart the build in Travis UI to continue from cache.' + fi + + set +e + + eval ' + function install_run {'\ + "$(if [ -n "$MESSAGE" ]; then + echo ' echo -e "\n'"$MESSAGE"'\n"' + fi)"\ + ' + # Travis runs user scripts via `eval` i.e. in the same shell process. + # So have to unset errexit in order to get to cache save stage + set +e + return '"$EXIT_CODE"' + }' +} + function pre_build_osx { local repo_dir=$(abspath ${1:-$REPO_DIR}) local build_dir="$repo_dir/opencv/build" local num_cpus=$(sysctl -n hw.ncpu) num_cpus=${num_cpus:-4} + local travis_start_time=$(($TRAVIS_TIMER_START_TIME/10**9)) + local time_limit=$((45*60)) cd "$repo_dir" git submodule sync git submodule update --init --recursive opencv git submodule update --init --recursive opencv_contrib - pip install scikit-build - pip install numpy - if [ ! -d "$build_dir" ]; then mkdir "$build_dir" fi @@ -127,13 +154,22 @@ function pre_build_osx { ) for m in "${CV_MODULES[@]}"; do if make help | grep -w "$m"; then + # Check time limit (3min should be enough for a module to built) + local projected_time=$(($(date +%s) - travis_start_time + 3 * 60)) + if [ $projected_time -ge $time_limit ]; then + if [ -n "$USE_CCACHE" ]; then ccache -s; fi + goto_exit + return 1 + fi make -j${num_cpus} "$m" + local elapsed_time=$(($(date +%s) - travis_start_time)) + echo "Elapsed time: "$((elapsed_time/60))"m (${elapsed_time}s)" fi done make -j${num_cpus} # Print ccache stats - ccache -s + if [ -n "$USE_CCACHE" ]; then ccache -s; fi } function build_osx { @@ -170,9 +206,13 @@ function build_osx { function build_bdist_osx_wheel { local repo_dir=$(abspath ${1:-$REPO_DIR}) [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 - pre_build_osx "$repo_dir" - if [ -n "$BUILD_DEPENDS" ]; then - pip install $(pip_opts) $BUILD_DEPENDS - fi + local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) + start_spinner + if [ -n "$(is_function "pre_build")" ]; then pre_build; fi + stop_spinner + pip install scikit-build + pip install numpy + pre_build_osx "$repo_dir" || return $? build_osx "$repo_dir" + repair_wheelhouse "$wheelhouse" } From 335990723ad1e43af25a4fccc09e0a09002ca057 Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Tue, 26 Jan 2021 22:19:57 +0300 Subject: [PATCH 6/7] Fix incorrect paths for delocate --- travis_osx_build.sh | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/travis_osx_build.sh b/travis_osx_build.sh index f09e1b37..c1574ae4 100644 --- a/travis_osx_build.sh +++ b/travis_osx_build.sh @@ -44,7 +44,7 @@ function pre_build_osx { local num_cpus=$(sysctl -n hw.ncpu) num_cpus=${num_cpus:-4} local travis_start_time=$(($TRAVIS_TIMER_START_TIME/10**9)) - local time_limit=$((45*60)) + local time_limit=$((46*60)) cd "$repo_dir" git submodule sync @@ -88,9 +88,6 @@ function pre_build_osx { ) fi - # Clear ccache stats - ccache -z - # Configure build cd "$build_dir" cmake "${CMAKE_OPTS[@]}" .. @@ -151,13 +148,13 @@ function pre_build_osx { opencv_alphamat opencv_stitching opencv_gapi + all ) for m in "${CV_MODULES[@]}"; do if make help | grep -w "$m"; then - # Check time limit (3min should be enough for a module to built) + # Check time limit (3 min should be enough for a module build) local projected_time=$(($(date +%s) - travis_start_time + 3 * 60)) if [ $projected_time -ge $time_limit ]; then - if [ -n "$USE_CCACHE" ]; then ccache -s; fi goto_exit return 1 fi @@ -166,10 +163,6 @@ function pre_build_osx { echo "Elapsed time: "$((elapsed_time/60))"m (${elapsed_time}s)" fi done - make -j${num_cpus} - - # Print ccache stats - if [ -n "$USE_CCACHE" ]; then ccache -s; fi } function build_osx { @@ -207,12 +200,22 @@ function build_bdist_osx_wheel { local repo_dir=$(abspath ${1:-$REPO_DIR}) [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1 local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse}) + start_spinner if [ -n "$(is_function "pre_build")" ]; then pre_build; fi stop_spinner + pip install scikit-build pip install numpy + + if [ -n "$USE_CCACHE" ]; then ccache -z; fi + pre_build_osx "$repo_dir" || return $? + + if [ -n "$USE_CCACHE" ]; then ccache -s; fi + build_osx "$repo_dir" + + cp "$repo_dir"/dist/*.whl "$wheelhouse" repair_wheelhouse "$wheelhouse" } From ad0668838a3446ceddecd3ab7f4bb68435e368d0 Mon Sep 17 00:00:00 2001 From: Igor Murzov Date: Wed, 27 Jan 2021 01:14:16 +0300 Subject: [PATCH 7/7] Test deployment to PyPI --- .travis.yml | 18 +++++++++--------- appveyor.yml | 11 +++++------ setup-osx.py | 12 ++++++------ setup.py | 12 ++++++------ 4 files changed, 26 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index fd4d4170..d4c8c42e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -964,26 +964,26 @@ after_success: | if [[ $ENABLE_CONTRIB == 0 ]]; then if [[ $ENABLE_HEADLESS == 0 ]]; then - echo "This is default build. Deployment will be done to to PyPI entry opencv-python." + echo "This is default build. Deployment will be done to PyPI entry opencv-python." else - echo "This is headless contrib build. Deployment will be done to to PyPI entry opencv-python-headless." + echo "This is headless contrib build. Deployment will be done to PyPI entry opencv-python-headless." fi else if [[ $ENABLE_HEADLESS == 0 ]]; then - echo "This is contrib build. Deployment will be done to to PyPI entry opencv-contrib-python." + echo "This is contrib build. Deployment will be done to PyPI entry opencv-contrib-python." else - echo "This is headless contrib build. Deployment will be done to to PyPI entry opencv-contrib-python-headless." + echo "This is headless contrib build. Deployment will be done to PyPI entry opencv-contrib-python-headless." fi fi if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then if [[ $SDIST == 1 ]]; then pip install twine - twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/dist/opencv* + twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/dist/openc* else pip install --user twine pip install --user --upgrade six - twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/opencv* + twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/openc* fi fi @@ -996,9 +996,9 @@ after_success: | /usr/bin/python -m pip install --user -U -I twine if [[ $SDIST == 1 ]]; then - /usr/bin/python -m twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/dist/opencv* + /usr/bin/python -m twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/dist/openc* else - /usr/bin/python -m twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/opencv* + /usr/bin/python -m twine upload -u ${PYPI_USER} -p ${PASS} --skip-existing ${TRAVIS_BUILD_DIR}/wheelhouse/openc* fi fi @@ -1019,7 +1019,7 @@ after_success: | if [[ $SDIST == 1 ]]; then az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/dist --pattern *.gz else - az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/wheelhouse --pattern opencv*.whl + az storage blob upload-batch -d ${TRAVIS_COMMIT} -s ${TRAVIS_BUILD_DIR}/wheelhouse --pattern openc*.whl fi fi diff --git a/appveyor.yml b/appveyor.yml index c79e7de4..fb28aa76 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,9 +1,8 @@ environment: USER: - secure: fXgF9uyy6sT0JoVOR7BoqA== - + secure: faDqtT21igTqQlsKgq9+UA== PASS: - secure: 0bXSOVjf9x8L7nErTivu92TF1FwNosTjFJQPmxp8Dys= + secure: LARTd04b3VIW+7ADTEP7Lw== matrix: - PYTHON: "C:\\Python36" @@ -158,7 +157,7 @@ before_test: cd ${Env:APPVEYOR_BUILD_FOLDER}\tests $env:PYTHONWARNINGS = "ignore:::pip._internal.cli.base_command" - &"${Env:PYTHON}\\python.exe" -m pip install --user --no-warn-script-location (ls "../dist/opencv_*.whl") + &"${Env:PYTHON}\\python.exe" -m pip install --user --no-warn-script-location (ls "../dist/opencovis_*.whl") if ($LastExitCode -ne 0) {throw $LastExitCode} test_script: @@ -168,7 +167,7 @@ test_script: "%PYTHON%\\python.exe" -m unittest test artifacts: -- path: dist\opencv*.whl +- path: dist\opencovis*.whl name: wheels deploy_script: @@ -194,7 +193,7 @@ deploy_script: } &"${Env:PYTHON}\\python.exe" -m pip install twine - &"${Env:PYTHON}\\python.exe" -m twine upload -u ${Env:USER} -p ${Env:PASS} --skip-existing dist/opencv* + &"${Env:PYTHON}\\python.exe" -m twine upload -u ${Env:USER} -p ${Env:PASS} --skip-existing dist/opencovis* } else { echo "Tag not set, deployment skipped." diff --git a/setup-osx.py b/setup-osx.py index 5af8e429..c128ca93 100644 --- a/setup-osx.py +++ b/setup-osx.py @@ -45,16 +45,16 @@ def main(): build_contrib = version["contrib"] build_headless = version["headless"] - package_name = "opencv-python" + package_name = "opencovis-python" if build_contrib and not build_headless: - package_name = "opencv-contrib-python" + package_name = "opencovis-contrib-python" if build_contrib and build_headless: - package_name = "opencv-contrib-python-headless" + package_name = "opencovis-contrib-python-headless" if build_headless and not build_contrib: - package_name = "opencv-python-headless" + package_name = "opencovis-python-headless" long_description = io.open("README.md", encoding="utf-8").read() @@ -70,7 +70,7 @@ def main(): setuptools.setup( name=package_name, version=package_version, - url="https://github.com/skvark/opencv-python", + url="https://github.com/GArik/opencv-python", license="MIT", description="Wrapper package for OpenCV python bindings.", long_description=long_description, @@ -78,7 +78,7 @@ def main(): packages=packages, package_data=package_data, include_package_data=True, - maintainer="Olli-Pekka Heinisuo", + maintainer="Igor Murzov", ext_modules=EmptyListWithLength(), install_requires=numpy_version, python_requires=">=3.6", diff --git a/setup.py b/setup.py index 67f22c60..a5074666 100644 --- a/setup.py +++ b/setup.py @@ -67,16 +67,16 @@ def main(): # https://stackoverflow.com/questions/1405913/python-32bit-or-64bit-mode x64 = sys.maxsize > 2 ** 32 - package_name = "opencv-python" + package_name = "opencovis-python" if build_contrib and not build_headless: - package_name = "opencv-contrib-python" + package_name = "opencovis-contrib-python" if build_contrib and build_headless: - package_name = "opencv-contrib-python-headless" + package_name = "opencovis-contrib-python-headless" if build_headless and not build_contrib: - package_name = "opencv-python-headless" + package_name = "opencovis-python-headless" long_description = io.open("README.md", encoding="utf-8").read() @@ -217,14 +217,14 @@ def main(): skbuild.setup( name=package_name, version=package_version, - url="https://github.com/skvark/opencv-python", + url="https://github.com/GArik/opencv-python", license="MIT", description="Wrapper package for OpenCV python bindings.", long_description=long_description, long_description_content_type="text/markdown", packages=packages, package_data=package_data, - maintainer="Olli-Pekka Heinisuo", + maintainer="Igor Murzov", ext_modules=EmptyListWithLength(), install_requires=numpy_version, python_requires=">=3.6",