Skip to content

[3.12] Convert change detection to a Python script (GH-129627) #130370

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 21, 2025
Merged

[3.12] Convert change detection to a Python script (GH-129627) #130370

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0ce2b94
[3.12] Convert change detection to a Python script (GH-129627)
AA-Turner Feb 5, 2025
e17a76b
Fix test_hypothesis
hugovk Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 38 additions & 44 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,32 +22,32 @@ env:
FORCE_COLOR: 1

jobs:
check_source:
build-context:
name: Change detection
# To use boolean outputs from this job, parse them as JSON.
# Here's some examples:
#
# if: fromJSON(needs.check_source.outputs.run-docs)
# if: fromJSON(needs.build-context.outputs.run-docs)
#
# ${{
# fromJSON(needs.check_source.outputs.run_tests)
# fromJSON(needs.build-context.outputs.run-tests)
# && 'truthy-branch'
# || 'falsy-branch'
# }}
#
uses: ./.github/workflows/reusable-change-detection.yml
uses: ./.github/workflows/reusable-context.yml

check-docs:
name: Docs
needs: check_source
if: fromJSON(needs.check_source.outputs.run-docs)
needs: build-context
if: fromJSON(needs.build-context.outputs.run-docs)
uses: ./.github/workflows/reusable-docs.yml

check_abi:
name: 'Check if the ABI has changed'
runs-on: ubuntu-22.04
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
steps:
- uses: actions/checkout@v4
with:
Expand Down Expand Up @@ -96,8 +96,8 @@ jobs:
container:
image: ghcr.io/python/autoconf:2024.10.16.11360930377
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
steps:
- name: Install Git
run: |
Expand Down Expand Up @@ -137,8 +137,8 @@ jobs:
# reproducible: to get the same tools versions (autoconf, aclocal, ...)
runs-on: ubuntu-24.04
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
steps:
- uses: actions/checkout@v4
with:
Expand All @@ -153,7 +153,7 @@ jobs:
with:
path: config.cache
# Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }}
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }}
- name: Install dependencies
run: sudo ./.github/workflows/posix-deps-apt.sh
- name: Add ccache to PATH
Expand Down Expand Up @@ -199,8 +199,8 @@ jobs:
name: >-
Windows
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
needs: check_source
if: fromJSON(needs.check_source.outputs.run_tests)
needs: build-context
if: fromJSON(needs.build-context.outputs.run-tests)
strategy:
matrix:
arch:
Expand All @@ -218,8 +218,8 @@ jobs:
build_windows_msi:
name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category
Windows MSI${{ '' }}
needs: check_source
if: fromJSON(needs.check_source.outputs.run-win-msi)
needs: build-context
if: fromJSON(needs.build-context.outputs.run-windows-msi)
strategy:
matrix:
arch:
Expand All @@ -234,8 +234,8 @@ jobs:
name: >-
macOS
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
strategy:
fail-fast: false
matrix:
Expand All @@ -260,32 +260,32 @@ jobs:
free-threading: true
uses: ./.github/workflows/reusable-macos.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
config_hash: ${{ needs.build-context.outputs.config-hash }}
free-threading: ${{ matrix.free-threading }}
os: ${{ matrix.os }}

build_ubuntu:
name: >-
Ubuntu
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
strategy:
matrix:
free-threading:
- false
# - true
uses: ./.github/workflows/reusable-ubuntu.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
config_hash: ${{ needs.build-context.outputs.config-hash }}
free-threading: ${{ matrix.free-threading }}

build_ubuntu_ssltests:
name: 'Ubuntu SSL tests with OpenSSL'
runs-on: ${{ matrix.os }}
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
strategy:
fail-fast: false
matrix:
Expand All @@ -306,7 +306,7 @@ jobs:
uses: actions/cache@v4
with:
path: config.cache
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }}
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install dependencies
Expand Down Expand Up @@ -345,8 +345,8 @@ jobs:
name: "Hypothesis tests on Ubuntu"
runs-on: ubuntu-24.04
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
env:
OPENSSL_VER: 3.0.15
PYTHONSTRICTEXTENSIONBUILD: 1
Expand Down Expand Up @@ -393,7 +393,7 @@ jobs:
uses: actions/cache@v4
with:
path: ${{ env.CPYTHON_BUILDDIR }}/config.cache
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }}
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
- name: Configure CPython out-of-tree
working-directory: ${{ env.CPYTHON_BUILDDIR }}
run: |
Expand Down Expand Up @@ -460,8 +460,8 @@ jobs:
name: 'Address sanitizer'
runs-on: ubuntu-22.04
timeout-minutes: 60
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
strategy:
matrix:
os: [ubuntu-24.04]
Expand All @@ -479,7 +479,7 @@ jobs:
uses: actions/cache@v4
with:
path: config.cache
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.check_source.outputs.config_hash }}
key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
- name: Register gcc problem matcher
run: echo "::add-matcher::.github/problem-matchers/gcc.json"
- name: Install dependencies
Expand Down Expand Up @@ -523,23 +523,23 @@ jobs:
name: >-
Thread sanitizer
${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
needs: check_source
if: needs.check_source.outputs.run_tests == 'true'
needs: build-context
if: needs.build-context.outputs.run-tests == 'true'
strategy:
matrix:
free-threading:
- false
uses: ./.github/workflows/reusable-tsan.yml
with:
config_hash: ${{ needs.check_source.outputs.config_hash }}
config_hash: ${{ needs.build-context.outputs.config-hash }}
free-threading: ${{ matrix.free-threading }}

all-required-green: # This job does nothing and is only used for the branch protection
name: All required checks pass
if: always()

needs:
- check_source # Transitive dependency, needed to access `run_tests` value
- build-context # Transitive dependency, needed to access `run-tests` value
- check-docs
- check_autoconf_regen
- check_generated_files
Expand All @@ -564,14 +564,14 @@ jobs:
test_hypothesis,
allowed-skips: >-
${{
!fromJSON(needs.check_source.outputs.run-docs)
!fromJSON(needs.build-context.outputs.run-docs)
&& '
check-docs,
'
|| ''
}}
${{
needs.check_source.outputs.run_tests != 'true'
needs.build-context.outputs.run-tests != 'true'
&& '
check_autoconf_regen,
check_generated_files,
Expand All @@ -581,12 +581,6 @@ jobs:
build_windows,
build_asan,
build_tsan,
'
|| ''
}}
${{
!fromJSON(needs.check_source.outputs.run_hypothesis)
&& '
test_hypothesis,
'
|| ''
Expand Down
100 changes: 100 additions & 0 deletions .github/workflows/reusable-context.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
name: Reusable build context

on: # yamllint disable-line rule:truthy
workflow_call:
outputs:
# Every referenced step MUST always set its output variable,
# either via ``Tools/build/compute-changes.py`` or in this workflow file.
# Boolean outputs (generally prefixed ``run-``) can then later be used
# safely through the following idiom in job conditionals and other
# expressions. Here's some examples:
#
# if: fromJSON(needs.build-context.outputs.run-tests)
#
# ${{
# fromJSON(needs.build-context.outputs.run-tests)
# && 'truthy-branch'
# || 'falsy-branch'
# }}
#
config-hash:
description: Config hash value for use in cache keys
value: ${{ jobs.compute-changes.outputs.config-hash }} # str
run-docs:
description: Whether to build the docs
value: ${{ jobs.compute-changes.outputs.run-docs }} # bool
run-tests:
description: Whether to run the regular tests
value: ${{ jobs.compute-changes.outputs.run-tests }} # bool
run-windows-msi:
description: Whether to run the MSI installer smoke tests
value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool
run-ci-fuzz:
description: Whether to run the CIFuzz job
value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool

jobs:
compute-changes:
name: Create context from changed files
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
config-hash: ${{ steps.config-hash.outputs.hash }}
run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
run-docs: ${{ steps.changes.outputs.run-docs }}
run-tests: ${{ steps.changes.outputs.run-tests }}
run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }}
steps:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3"

- run: >-
echo '${{ github.event_name }}'

- uses: actions/checkout@v4
with:
persist-credentials: false
ref: >-
${{
github.event_name == 'pull_request'
&& github.event.pull_request.head.sha
|| ''
}}

# Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721
- name: Fetch commits to get branch diff
if: github.event_name == 'pull_request'
run: |
set -eux

# Fetch enough history to find a common ancestor commit (aka merge-base):
git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \
--no-tags --prune --no-recurse-submodules

# This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from):
COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" )
DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" )

# Get all commits since that commit date from the base branch (eg: main):
git fetch origin "${refspec_base}" --shallow-since="${DATE}" \
--no-tags --prune --no-recurse-submodules
env:
branch_pr: 'origin/${{ github.event.pull_request.head.ref }}'
commits: ${{ github.event.pull_request.commits }}
refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}'
refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}'

# We only want to run tests on PRs when related files are changed,
# or when someone triggers a manual workflow run.
- name: Compute changed files
id: changes
run: python Tools/build/compute-changes.py
env:
GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}

- name: Compute hash for config cache key
id: config-hash
run: |
echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT"
Loading
Loading