From 74ba72c76697902d67c61266bb489a8560a10e94 Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Tue, 10 Mar 2020 23:12:32 -0400 Subject: [PATCH 1/4] STY BAD linting test --- azure-pipelines.yml | 18 +++++++++--------- sklearn/datasets/tests/test_base.py | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1aad015849b2e..bee73da2694a3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,6 +19,15 @@ jobs: displayName: Take ownership of conda installation - bash: conda create --name flake8_env --yes flake8 displayName: Install flake8 + - bash: | + if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[scipy-dev\] ]] || \ + [[ $BUILD_REASON == "Schedule" ]]; then + echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]true" + else + echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]false" + fi + name: gitCommitMessage + displayName: Determine to run scipy-dev - bash: | if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[lint\ skip\] ]]; then # skip linting @@ -29,15 +38,6 @@ jobs: ./build_tools/circle/linting.sh fi displayName: Run linting - - bash: | - if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[scipy-dev\] ]] || \ - [[ $BUILD_REASON == "Schedule" ]]; then - echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]true" - else - echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]false" - fi - name: gitCommitMessage - displayName: Determine to run scipy-dev - template: build_tools/azure/posix.yml parameters: diff --git a/sklearn/datasets/tests/test_base.py b/sklearn/datasets/tests/test_base.py index a58bdc9ed644d..c6dc342e9d9a9 100644 --- a/sklearn/datasets/tests/test_base.py +++ b/sklearn/datasets/tests/test_base.py @@ -9,6 +9,7 @@ import pytest import joblib +from sklearn.datasets import make_biclusters import numpy as np from sklearn.datasets import get_data_home From 03ebc974246e52f44e78feddacfc1f300d2710d3 Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Wed, 11 Mar 2020 00:39:30 -0400 Subject: [PATCH 2/4] TST Test fix --- azure-pipelines.yml | 189 ++++++++++++++++++++++---------------------- 1 file changed, 95 insertions(+), 94 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bee73da2694a3..8cf9c9700de3b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,15 +19,6 @@ jobs: displayName: Take ownership of conda installation - bash: conda create --name flake8_env --yes flake8 displayName: Install flake8 - - bash: | - if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[scipy-dev\] ]] || \ - [[ $BUILD_REASON == "Schedule" ]]; then - echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]true" - else - echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]false" - fi - name: gitCommitMessage - displayName: Determine to run scipy-dev - bash: | if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[lint\ skip\] ]]; then # skip linting @@ -38,6 +29,15 @@ jobs: ./build_tools/circle/linting.sh fi displayName: Run linting + - bash: | + if [[ $BUILD_SOURCEVERSIONMESSAGE =~ \[scipy-dev\] ]] || \ + [[ $BUILD_REASON == "Schedule" ]]; then + echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]true" + else + echo "##vso[task.setvariable variable=runScipyDev;isOutput=true]false" + fi + name: gitCommitMessage + displayName: Determine to run scipy-dev - template: build_tools/azure/posix.yml parameters: @@ -80,6 +80,7 @@ jobs: name: Linux vmImage: ubuntu-18.04 dependsOn: [linting] + condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting')) matrix: # Linux environment to test that scikit-learn can be built against # versions of numpy, scipy with ATLAS that comes with Ubuntu Bionic 18.04 @@ -90,91 +91,91 @@ jobs: JOBLIB_VERSION: '0.11' THREADPOOLCTL_VERSION: '2.0.0' # Linux + Python 3.6 build with OpenBLAS and without SITE_JOBLIB - py36_conda_openblas: - DISTRIB: 'conda' - PYTHON_VERSION: '3.6' - BLAS: 'openblas' - NUMPY_VERSION: '1.13.3' - SCIPY_VERSION: '0.19.1' - PANDAS_VERSION: '*' - CYTHON_VERSION: '*' - # temporary pin pytest due to unknown failure with pytest 5.3 - PYTEST_VERSION: '5.2' - PILLOW_VERSION: '4.2.1' - MATPLOTLIB_VERSION: '2.1.1' - SCIKIT_IMAGE_VERSION: '*' - # latest version of joblib available in conda for Python 3.6 - JOBLIB_VERSION: '0.13.2' - THREADPOOLCTL_VERSION: '2.0.0' - COVERAGE: 'true' - # Linux environment to test the latest available dependencies and MKL. - # It runs tests requiring lightgbm, pandas and PyAMG. - pylatest_pip_openblas_pandas: - DISTRIB: 'conda-pip-latest' - PYTHON_VERSION: '3.8' - PYTEST_VERSION: '4.6.2' - COVERAGE: 'true' - CHECK_PYTEST_SOFT_DEPENDENCY: 'true' - TEST_DOCSTRINGS: 'true' - CHECK_WARNINGS: 'true' +# py36_conda_openblas: +# DISTRIB: 'conda' +# PYTHON_VERSION: '3.6' +# BLAS: 'openblas' +# NUMPY_VERSION: '1.13.3' +# SCIPY_VERSION: '0.19.1' +# PANDAS_VERSION: '*' +# CYTHON_VERSION: '*' +# # temporary pin pytest due to unknown failure with pytest 5.3 +# PYTEST_VERSION: '5.2' +# PILLOW_VERSION: '4.2.1' +# MATPLOTLIB_VERSION: '2.1.1' +# SCIKIT_IMAGE_VERSION: '*' +# # latest version of joblib available in conda for Python 3.6 +# JOBLIB_VERSION: '0.13.2' +# THREADPOOLCTL_VERSION: '2.0.0' +# COVERAGE: 'true' +# # Linux environment to test the latest available dependencies and MKL. +# # It runs tests requiring lightgbm, pandas and PyAMG. +# pylatest_pip_openblas_pandas: +# DISTRIB: 'conda-pip-latest' +# PYTHON_VERSION: '3.8' +# PYTEST_VERSION: '4.6.2' +# COVERAGE: 'true' +# CHECK_PYTEST_SOFT_DEPENDENCY: 'true' +# TEST_DOCSTRINGS: 'true' +# CHECK_WARNINGS: 'true' -- template: build_tools/azure/posix-32.yml - parameters: - name: Linux32 - vmImage: ubuntu-18.04 - dependsOn: [linting] - matrix: - py36_ubuntu_atlas_32bit: - DISTRIB: 'ubuntu-32' - PYTHON_VERSION: '3.6' - JOBLIB_VERSION: '0.13' - THREADPOOLCTL_VERSION: '2.0.0' +# - template: build_tools/azure/posix-32.yml +# parameters: +# name: Linux32 +# vmImage: ubuntu-18.04 +# dependsOn: [linting] +# matrix: +# py36_ubuntu_atlas_32bit: +# DISTRIB: 'ubuntu-32' +# PYTHON_VERSION: '3.6' +# JOBLIB_VERSION: '0.13' +# THREADPOOLCTL_VERSION: '2.0.0' -- template: build_tools/azure/posix.yml - parameters: - name: macOS - vmImage: macOS-10.14 - dependsOn: [linting] - matrix: - pylatest_conda_mkl: - DISTRIB: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - NUMPY_VERSION: '*' - SCIPY_VERSION: '*' - CYTHON_VERSION: '*' - PILLOW_VERSION: '*' - PYTEST_VERSION: '*' - JOBLIB_VERSION: '*' - THREADPOOLCTL_VERSION: '2.0.0' - COVERAGE: 'true' - pylatest_conda_mkl_no_openmp: - DISTRIB: 'conda' - PYTHON_VERSION: '*' - BLAS: 'mkl' - NUMPY_VERSION: '*' - SCIPY_VERSION: '*' - CYTHON_VERSION: '*' - PILLOW_VERSION: '*' - PYTEST_VERSION: '*' - JOBLIB_VERSION: '*' - THREADPOOLCTL_VERSION: '2.0.0' - COVERAGE: 'true' - SKLEARN_TEST_NO_OPENMP: 'true' - SKLEARN_SKIP_OPENMP_TEST: 'true' +# - template: build_tools/azure/posix.yml +# parameters: +# name: macOS +# vmImage: macOS-10.14 +# dependsOn: [linting] +# matrix: +# pylatest_conda_mkl: +# DISTRIB: 'conda' +# PYTHON_VERSION: '*' +# BLAS: 'mkl' +# NUMPY_VERSION: '*' +# SCIPY_VERSION: '*' +# CYTHON_VERSION: '*' +# PILLOW_VERSION: '*' +# PYTEST_VERSION: '*' +# JOBLIB_VERSION: '*' +# THREADPOOLCTL_VERSION: '2.0.0' +# COVERAGE: 'true' +# pylatest_conda_mkl_no_openmp: +# DISTRIB: 'conda' +# PYTHON_VERSION: '*' +# BLAS: 'mkl' +# NUMPY_VERSION: '*' +# SCIPY_VERSION: '*' +# CYTHON_VERSION: '*' +# PILLOW_VERSION: '*' +# PYTEST_VERSION: '*' +# JOBLIB_VERSION: '*' +# THREADPOOLCTL_VERSION: '2.0.0' +# COVERAGE: 'true' +# SKLEARN_TEST_NO_OPENMP: 'true' +# SKLEARN_SKIP_OPENMP_TEST: 'true' -- template: build_tools/azure/windows.yml - parameters: - name: Windows - vmImage: vs2017-win2016 - dependsOn: [linting] - matrix: - py37_conda_mkl: - PYTHON_VERSION: '3.7' - CHECK_WARNINGS: 'true' - PYTHON_ARCH: '64' - PYTEST_VERSION: '*' - COVERAGE: 'true' - py36_pip_openblas_32bit: - PYTHON_VERSION: '3.6' - PYTHON_ARCH: '32' +# - template: build_tools/azure/windows.yml +# parameters: +# name: Windows +# vmImage: vs2017-win2016 +# dependsOn: [linting] +# matrix: +# py37_conda_mkl: +# PYTHON_VERSION: '3.7' +# CHECK_WARNINGS: 'true' +# PYTHON_ARCH: '64' +# PYTEST_VERSION: '*' +# COVERAGE: 'true' +# py36_pip_openblas_32bit: +# PYTHON_VERSION: '3.6' +# PYTHON_ARCH: '32' From 0f85a5b821eb10ef6e18490c754822c6aa587521 Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Wed, 11 Mar 2020 00:43:40 -0400 Subject: [PATCH 3/4] TST Test fix --- azure-pipelines.yml | 173 ++++++++++++++++++------------------ build_tools/azure/posix.yml | 2 +- 2 files changed, 89 insertions(+), 86 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8cf9c9700de3b..047988e25f648 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -91,91 +91,94 @@ jobs: JOBLIB_VERSION: '0.11' THREADPOOLCTL_VERSION: '2.0.0' # Linux + Python 3.6 build with OpenBLAS and without SITE_JOBLIB -# py36_conda_openblas: -# DISTRIB: 'conda' -# PYTHON_VERSION: '3.6' -# BLAS: 'openblas' -# NUMPY_VERSION: '1.13.3' -# SCIPY_VERSION: '0.19.1' -# PANDAS_VERSION: '*' -# CYTHON_VERSION: '*' -# # temporary pin pytest due to unknown failure with pytest 5.3 -# PYTEST_VERSION: '5.2' -# PILLOW_VERSION: '4.2.1' -# MATPLOTLIB_VERSION: '2.1.1' -# SCIKIT_IMAGE_VERSION: '*' -# # latest version of joblib available in conda for Python 3.6 -# JOBLIB_VERSION: '0.13.2' -# THREADPOOLCTL_VERSION: '2.0.0' -# COVERAGE: 'true' -# # Linux environment to test the latest available dependencies and MKL. -# # It runs tests requiring lightgbm, pandas and PyAMG. -# pylatest_pip_openblas_pandas: -# DISTRIB: 'conda-pip-latest' -# PYTHON_VERSION: '3.8' -# PYTEST_VERSION: '4.6.2' -# COVERAGE: 'true' -# CHECK_PYTEST_SOFT_DEPENDENCY: 'true' -# TEST_DOCSTRINGS: 'true' -# CHECK_WARNINGS: 'true' + py36_conda_openblas: + DISTRIB: 'conda' + PYTHON_VERSION: '3.6' + BLAS: 'openblas' + NUMPY_VERSION: '1.13.3' + SCIPY_VERSION: '0.19.1' + PANDAS_VERSION: '*' + CYTHON_VERSION: '*' + # temporary pin pytest due to unknown failure with pytest 5.3 + PYTEST_VERSION: '5.2' + PILLOW_VERSION: '4.2.1' + MATPLOTLIB_VERSION: '2.1.1' + SCIKIT_IMAGE_VERSION: '*' + # latest version of joblib available in conda for Python 3.6 + JOBLIB_VERSION: '0.13.2' + THREADPOOLCTL_VERSION: '2.0.0' + COVERAGE: 'true' + # Linux environment to test the latest available dependencies and MKL. + # It runs tests requiring lightgbm, pandas and PyAMG. + pylatest_pip_openblas_pandas: + DISTRIB: 'conda-pip-latest' + PYTHON_VERSION: '3.8' + PYTEST_VERSION: '4.6.2' + COVERAGE: 'true' + CHECK_PYTEST_SOFT_DEPENDENCY: 'true' + TEST_DOCSTRINGS: 'true' + CHECK_WARNINGS: 'true' -# - template: build_tools/azure/posix-32.yml -# parameters: -# name: Linux32 -# vmImage: ubuntu-18.04 -# dependsOn: [linting] -# matrix: -# py36_ubuntu_atlas_32bit: -# DISTRIB: 'ubuntu-32' -# PYTHON_VERSION: '3.6' -# JOBLIB_VERSION: '0.13' -# THREADPOOLCTL_VERSION: '2.0.0' +- template: build_tools/azure/posix-32.yml + parameters: + name: Linux32 + vmImage: ubuntu-18.04 + dependsOn: [linting] + condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting')) + matrix: + py36_ubuntu_atlas_32bit: + DISTRIB: 'ubuntu-32' + PYTHON_VERSION: '3.6' + JOBLIB_VERSION: '0.13' + THREADPOOLCTL_VERSION: '2.0.0' -# - template: build_tools/azure/posix.yml -# parameters: -# name: macOS -# vmImage: macOS-10.14 -# dependsOn: [linting] -# matrix: -# pylatest_conda_mkl: -# DISTRIB: 'conda' -# PYTHON_VERSION: '*' -# BLAS: 'mkl' -# NUMPY_VERSION: '*' -# SCIPY_VERSION: '*' -# CYTHON_VERSION: '*' -# PILLOW_VERSION: '*' -# PYTEST_VERSION: '*' -# JOBLIB_VERSION: '*' -# THREADPOOLCTL_VERSION: '2.0.0' -# COVERAGE: 'true' -# pylatest_conda_mkl_no_openmp: -# DISTRIB: 'conda' -# PYTHON_VERSION: '*' -# BLAS: 'mkl' -# NUMPY_VERSION: '*' -# SCIPY_VERSION: '*' -# CYTHON_VERSION: '*' -# PILLOW_VERSION: '*' -# PYTEST_VERSION: '*' -# JOBLIB_VERSION: '*' -# THREADPOOLCTL_VERSION: '2.0.0' -# COVERAGE: 'true' -# SKLEARN_TEST_NO_OPENMP: 'true' -# SKLEARN_SKIP_OPENMP_TEST: 'true' +- template: build_tools/azure/posix.yml + parameters: + name: macOS + vmImage: macOS-10.14 + dependsOn: [linting] + condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting')) + matrix: + pylatest_conda_mkl: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + BLAS: 'mkl' + NUMPY_VERSION: '*' + SCIPY_VERSION: '*' + CYTHON_VERSION: '*' + PILLOW_VERSION: '*' + PYTEST_VERSION: '*' + JOBLIB_VERSION: '*' + THREADPOOLCTL_VERSION: '2.0.0' + COVERAGE: 'true' + pylatest_conda_mkl_no_openmp: + DISTRIB: 'conda' + PYTHON_VERSION: '*' + BLAS: 'mkl' + NUMPY_VERSION: '*' + SCIPY_VERSION: '*' + CYTHON_VERSION: '*' + PILLOW_VERSION: '*' + PYTEST_VERSION: '*' + JOBLIB_VERSION: '*' + THREADPOOLCTL_VERSION: '2.0.0' + COVERAGE: 'true' + SKLEARN_TEST_NO_OPENMP: 'true' + SKLEARN_SKIP_OPENMP_TEST: 'true' -# - template: build_tools/azure/windows.yml -# parameters: -# name: Windows -# vmImage: vs2017-win2016 -# dependsOn: [linting] -# matrix: -# py37_conda_mkl: -# PYTHON_VERSION: '3.7' -# CHECK_WARNINGS: 'true' -# PYTHON_ARCH: '64' -# PYTEST_VERSION: '*' -# COVERAGE: 'true' -# py36_pip_openblas_32bit: -# PYTHON_VERSION: '3.6' -# PYTHON_ARCH: '32' +- template: build_tools/azure/windows.yml + parameters: + name: Windows + vmImage: vs2017-win2016 + dependsOn: [linting] + condition: and(ne(variables['Build.Reason'], 'Schedule'), succeeded('linting')) + matrix: + py37_conda_mkl: + PYTHON_VERSION: '3.7' + CHECK_WARNINGS: 'true' + PYTHON_ARCH: '64' + PYTEST_VERSION: '*' + COVERAGE: 'true' + py36_pip_openblas_32bit: + PYTHON_VERSION: '3.6' + PYTHON_ARCH: '32' diff --git a/build_tools/azure/posix.yml b/build_tools/azure/posix.yml index 9efb0418278d2..c48e3644680bd 100644 --- a/build_tools/azure/posix.yml +++ b/build_tools/azure/posix.yml @@ -3,7 +3,7 @@ parameters: vmImage: '' matrix: [] dependsOn: [] - condition: ne(variables['Build.Reason'], 'Schedule') + condition: '' jobs: - job: ${{ parameters.name }} From 4720576e2076c071397d1619a2a3475f9f33528a Mon Sep 17 00:00:00 2001 From: Thomas J Fan Date: Wed, 11 Mar 2020 00:47:37 -0400 Subject: [PATCH 4/4] REV Revert change --- sklearn/datasets/tests/test_base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sklearn/datasets/tests/test_base.py b/sklearn/datasets/tests/test_base.py index c6dc342e9d9a9..a58bdc9ed644d 100644 --- a/sklearn/datasets/tests/test_base.py +++ b/sklearn/datasets/tests/test_base.py @@ -9,7 +9,6 @@ import pytest import joblib -from sklearn.datasets import make_biclusters import numpy as np from sklearn.datasets import get_data_home