NiPyAPI 1.0 Release - Apache NiFi 2.x Compatibility #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [ main, maint-0.x ] | |
| pull_request: | |
| branches: [ main, maint-0.x ] | |
| jobs: | |
| test-python-312-single-user: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] | |
| - name: Lint with flake8 and pylint | |
| run: make lint | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate certificates | |
| run: make certs | |
| - name: Start NiFi infrastructure (single-user) | |
| run: make up NIPYAPI_PROFILE=single-user | |
| - name: Wait for NiFi to be ready | |
| run: make wait-ready NIPYAPI_PROFILE=single-user | |
| - name: Run integration tests with coverage | |
| run: NIPYAPI_PROFILE=single-user PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest --cov=nipyapi --cov-report=xml --cov-report=term-missing | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| - name: Test distribution build and import | |
| run: make test-dist | |
| - name: Stop NiFi infrastructure | |
| if: always() | |
| run: make down | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| file: ./coverage.xml | |
| fail_ci_if_error: false | |
| verbose: true | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| test-python-311-single-user: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.11 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate certificates | |
| run: make certs | |
| - name: Start NiFi infrastructure (single-user) | |
| run: make up NIPYAPI_PROFILE=single-user | |
| - name: Wait for NiFi to be ready | |
| run: make wait-ready NIPYAPI_PROFILE=single-user | |
| - name: Run integration tests | |
| run: NIPYAPI_PROFILE=single-user PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| - name: Stop NiFi infrastructure | |
| if: always() | |
| run: make down | |
| test-python-310-single-user: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate certificates | |
| run: make certs | |
| - name: Start NiFi infrastructure (single-user) | |
| run: make up NIPYAPI_PROFILE=single-user | |
| - name: Wait for NiFi to be ready | |
| run: make wait-ready NIPYAPI_PROFILE=single-user | |
| - name: Run integration tests | |
| run: NIPYAPI_PROFILE=single-user PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| - name: Stop NiFi infrastructure | |
| if: always() | |
| run: make down | |
| test-python-39-single-user: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.9 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.9" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate certificates | |
| run: make certs | |
| - name: Start NiFi infrastructure (single-user) | |
| run: make up NIPYAPI_PROFILE=single-user | |
| - name: Wait for NiFi to be ready | |
| run: make wait-ready NIPYAPI_PROFILE=single-user | |
| - name: Run integration tests | |
| run: NIPYAPI_PROFILE=single-user PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| - name: Stop NiFi infrastructure | |
| if: always() | |
| run: make down | |
| test-python-312-secure-ldap: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate certificates | |
| run: make certs | |
| - name: Start NiFi infrastructure (secure-ldap) | |
| run: make up NIPYAPI_PROFILE=secure-ldap | |
| - name: Wait for containers to stabilize | |
| run: sleep 5 && docker ps -a | |
| - name: Debug SSL environment | |
| run: | | |
| python -c "import ssl; print('OpenSSL version:', ssl.OPENSSL_VERSION)" | |
| python -c "import ssl; print('SSL context flags:', ssl.create_default_context().verify_flags)" | |
| - name: Debug environment variables | |
| run: | | |
| echo "=== Environment Variables ===" | |
| echo "NIFI_VERIFY_SSL: ${NIFI_VERIFY_SSL:-unset}" | |
| echo "REGISTRY_VERIFY_SSL: ${REGISTRY_VERIFY_SSL:-unset}" | |
| echo "NIPYAPI_CHECK_HOSTNAME: ${NIPYAPI_CHECK_HOSTNAME:-unset}" | |
| echo "=== Testing getenv_bool parsing ===" | |
| python -c " | |
| import os | |
| os.environ['NIFI_VERIFY_SSL'] = '0' | |
| os.environ['REGISTRY_VERIFY_SSL'] = '0' | |
| os.environ['NIPYAPI_CHECK_HOSTNAME'] = '0' | |
| import nipyapi | |
| print('NIFI_VERIFY_SSL via getenv_bool:', nipyapi.utils.getenv_bool('NIFI_VERIFY_SSL')) | |
| print('REGISTRY_VERIFY_SSL via getenv_bool:', nipyapi.utils.getenv_bool('REGISTRY_VERIFY_SSL')) | |
| print('NIPYAPI_CHECK_HOSTNAME via getenv_bool:', nipyapi.utils.getenv_bool('NIPYAPI_CHECK_HOSTNAME')) | |
| " | |
| env: | |
| NIFI_VERIFY_SSL: "0" | |
| REGISTRY_VERIFY_SSL: "0" | |
| NIPYAPI_CHECK_HOSTNAME: "0" | |
| - name: Wait for NiFi to be ready | |
| run: make wait-ready NIPYAPI_PROFILE=secure-ldap | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| NIFI_VERIFY_SSL: "0" | |
| REGISTRY_VERIFY_SSL: "0" | |
| NIPYAPI_CHECK_HOSTNAME: "0" | |
| - name: Run integration tests | |
| run: NIPYAPI_PROFILE=secure-ldap PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| NIFI_VERIFY_SSL: "0" | |
| REGISTRY_VERIFY_SSL: "0" | |
| NIPYAPI_CHECK_HOSTNAME: "0" | |
| - name: Dump running containers and logs on failure | |
| if: failure() | |
| run: | | |
| echo "=== Container Status ===" | |
| docker ps -a | |
| echo "=== Container Logs ===" | |
| for container in $(docker ps -aq); do | |
| echo "--- Logs for container: $container ---" | |
| docker logs "$container" 2>&1 || echo "Failed to get logs for container $container" | |
| echo "" | |
| done | |
| - name: Stop NiFi infrastructure | |
| if: always() | |
| run: make down | |
| test-python-312-secure-mtls: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.12 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e .[dev] | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Generate certificates | |
| run: make certs | |
| - name: Start NiFi infrastructure (secure-mtls) | |
| run: make up NIPYAPI_PROFILE=secure-mtls | |
| - name: Wait for containers to stabilize | |
| run: sleep 5 && docker ps -a | |
| - name: Debug SSL environment | |
| run: | | |
| python -c "import ssl; print('OpenSSL version:', ssl.OPENSSL_VERSION)" | |
| python -c "import ssl; print('SSL context flags:', ssl.create_default_context().verify_flags)" | |
| - name: Debug environment variables | |
| run: | | |
| echo "=== Environment Variables ===" | |
| echo "NIFI_VERIFY_SSL: ${NIFI_VERIFY_SSL:-unset}" | |
| echo "REGISTRY_VERIFY_SSL: ${REGISTRY_VERIFY_SSL:-unset}" | |
| echo "NIPYAPI_CHECK_HOSTNAME: ${NIPYAPI_CHECK_HOSTNAME:-unset}" | |
| echo "=== Testing getenv_bool parsing ===" | |
| python -c " | |
| import os | |
| os.environ['NIFI_VERIFY_SSL'] = '0' | |
| os.environ['REGISTRY_VERIFY_SSL'] = '0' | |
| os.environ['NIPYAPI_CHECK_HOSTNAME'] = '0' | |
| import nipyapi | |
| print('NIFI_VERIFY_SSL via getenv_bool:', nipyapi.utils.getenv_bool('NIFI_VERIFY_SSL')) | |
| print('REGISTRY_VERIFY_SSL via getenv_bool:', nipyapi.utils.getenv_bool('REGISTRY_VERIFY_SSL')) | |
| print('NIPYAPI_CHECK_HOSTNAME via getenv_bool:', nipyapi.utils.getenv_bool('NIPYAPI_CHECK_HOSTNAME')) | |
| " | |
| env: | |
| NIFI_VERIFY_SSL: "0" | |
| REGISTRY_VERIFY_SSL: "0" | |
| NIPYAPI_CHECK_HOSTNAME: "0" | |
| - name: Wait for NiFi to be ready | |
| run: make wait-ready NIPYAPI_PROFILE=secure-mtls | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| NIFI_VERIFY_SSL: "0" | |
| REGISTRY_VERIFY_SSL: "0" | |
| NIPYAPI_CHECK_HOSTNAME: "0" | |
| - name: Run integration tests | |
| run: NIPYAPI_PROFILE=secure-mtls PYTHONPATH=${{ github.workspace }}:$PYTHONPATH pytest | |
| env: | |
| PYTHONHTTPSVERIFY: "0" | |
| CURL_CA_BUNDLE: "" | |
| REQUESTS_CA_BUNDLE: "" | |
| NIFI_VERIFY_SSL: "0" | |
| REGISTRY_VERIFY_SSL: "0" | |
| NIPYAPI_CHECK_HOSTNAME: "0" | |
| - name: Dump running containers and logs on failure | |
| if: failure() | |
| run: | | |
| echo "=== Container Status ===" | |
| docker ps -a | |
| echo "=== Container Logs ===" | |
| for container in $(docker ps -aq); do | |
| echo "--- Logs for container: $container ---" | |
| docker logs "$container" 2>&1 || echo "Failed to get logs for container $container" | |
| echo "" | |
| done | |
| - name: Stop NiFi infrastructure | |
| if: always() | |
| run: make down |