From 819da3b62e0f01683c470ba3830c230a4fdc64f9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 3 Feb 2025 23:15:24 +0000 Subject: [PATCH 01/14] Convert change detection to a Python script --- .../workflows/reusable-change-detection.yml | 149 ++++--------- Tools/build/compute-changes.py | 199 ++++++++++++++++++ 2 files changed, 242 insertions(+), 106 deletions(-) create mode 100644 Tools/build/compute-changes.py diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index c08c0cb8873f12..5f43ead83617ee 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -52,122 +52,59 @@ jobs: timeout-minutes: 10 outputs: config-hash: ${{ steps.config-hash.outputs.hash }} - run-cifuzz: ${{ steps.check.outputs.run-cifuzz }} - run-docs: ${{ steps.docs-changes.outputs.run-docs }} - run-hypothesis: ${{ steps.check.outputs.run-hypothesis }} - run-tests: ${{ steps.check.outputs.run-tests }} - run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi }} + run-cifuzz: ${{ steps.changes.outputs.run-cifuzz }} + run-docs: ${{ steps.changes.outputs.run-docs }} + run-hypothesis: ${{ steps.changes.outputs.run-hypothesis }} + run-tests: ${{ steps.changes.outputs.run-tests }} + run-win-msi: ${{ steps.changes.outputs.run-win-msi }} steps: + - uses: actions/setup-python@v5 + with: + python-version: "3" + - run: >- echo '${{ github.event_name }}' + - uses: actions/checkout@v4 with: persist-credentials: false - - name: Check for source changes - id: check + ref: >- + ${{ + github.event_name == 'pull_request' + && github.event.pull_request.head.sha + || '' + }} + + # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721 + - name: Fetch commits to get branch diff + if: github.event_name == 'pull_request' run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run-tests=true" >> "$GITHUB_OUTPUT" - else - git fetch origin "$GITHUB_BASE_REF" --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - grep_ignore_args=( - # file extensions - -e '\.md$' - -e '\.rst$' - # top-level folders - -e '^Doc/' - -e '^Misc/' - # configuration files - -e '^\.github/CODEOWNERS$' - -e '^\.pre-commit-config\.yaml$' - -e '\.ruff\.toml$' - -e 'mypy\.ini$' - ) - git diff --name-only "origin/$GITHUB_BASE_REF.." \ - | grep -qvE "${grep_ignore_args[@]}" \ - && echo "run-tests=true" >> "$GITHUB_OUTPUT" || true - fi + set -eux + + # Fetch enough history to find a common ancestor commit (aka merge-base): + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ + --no-tags --prune --no-recurse-submodules + + # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) + DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) - # Check if we should run hypothesis tests - GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo "$GIT_BRANCH" - if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then - echo "Branch too old for hypothesis tests" - echo "run-hypothesis=false" >> "$GITHUB_OUTPUT" - else - echo "Run hypothesis tests" - echo "run-hypothesis=true" >> "$GITHUB_OUTPUT" - fi + # Get all commits since that commit date from the base branch (eg: master or main): + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ + --no-tags --prune --no-recurse-submodules + env: + branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} + refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' + refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' + + # We only want to run tests on PRs when related files are changed, + # or when someone triggers a manual workflow run. + - name: Compute changed files + id: changes + run: python Tools/build/compute-changes.py - # oss-fuzz maintains a configuration for fuzzing the main branch of - # CPython, so CIFuzz should be run only for code that is likely to be - # merged into the main branch; compatibility with older branches may - # be broken. - FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' - if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only "origin/$GITHUB_BASE_REF.." | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then - # The tests are pretty slow so they are executed only for PRs - # changing relevant files. - echo "Run CIFuzz tests" - echo "run-cifuzz=true" >> "$GITHUB_OUTPUT" - else - echo "Branch too old for CIFuzz tests; or no C files were changed" - echo "run-cifuzz=false" >> "$GITHUB_OUTPUT" - fi - name: Compute hash for config cache key id: config-hash run: | echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - # We only want to run this on PRs when related files are changed, - # or when user triggers manual workflow run. - if: >- - ( - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - ) || github.event_name == 'workflow_dispatch' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" - - name: Get a list of the MSI installer-related files - if: github.event_name == 'pull_request' - id: changed-win-msi-files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: | - Tools/msi/** - .github/workflows/reusable-windows-msi.yml - format: csv # works for paths with spaces - - name: Check for changes in MSI installer-related files - # We only want to run this on PRs when related files are changed, - # or when user triggers manual workflow run. - if: >- - ( - github.event_name == 'pull_request' - && steps.changed-win-msi-files.outputs.added_modified_renamed != '' - ) || github.event_name == 'workflow_dispatch' - id: win-msi-changes - run: | - echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py new file mode 100644 index 00000000000000..42fbec7bc5283b --- /dev/null +++ b/Tools/build/compute-changes.py @@ -0,0 +1,199 @@ +"""Determine which GitHub Actions workflows to run. + +Called by ``.github/workflows/reusable-change-detection.yml``. +We only want to run tests on PRs when related files are changed, +or when someone triggers a manual workflow run. +This improves developer experience by not doing (slow) +unnecessary work in GHA, and saves CI resources. +""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Set + +GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") +GITHUB_WORKFLOWS_PATH = Path(".github/workflows") +CONFIGURATION_FILE_NAMES = frozenset({ + ".pre-commit-config.yaml", + ".ruff.toml", + "mypy.ini", +}) +SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"}) +SUFFIXES_C_OR_CPP = frozenset({".c", ".h", ".cpp"}) + + +@dataclass(kw_only=True, slots=True) +class Outputs: + run_ci_fuzz: bool = False + run_docs: bool = False + run_hypothesis: bool = False + run_tests: bool = False + run_windows_msi: bool = False + + +def compute_changes(): + target_branch, head_branch = git_branches() + if target_branch and head_branch: + # Getting changed files only makes sense on a pull request + files = get_changed_files( + f"origin/{target_branch}", f"origin/{head_branch}" + ) + outputs = process_changed_files(files) + else: + # Otherwise, just run the tests + outputs = Outputs(run_tests=True) + outputs = process_target_branch(outputs, target_branch) + + if outputs.run_tests: + print("Run tests") + + if outputs.run_hypothesis: + print("Run hypothesis tests") + + if outputs.run_ci_fuzz: + print("Run CIFuzz tests") + else: + print("Branch too old for CIFuzz tests; or no C files were changed") + + if outputs.run_docs: + print("Build documentation") + + if outputs.run_windows_msi: + print("Build Windows MSI") + + print(outputs) + + write_github_output(outputs) + + +def git_branches() -> tuple[str, str]: + target_branch = os.environ.get("GITHUB_BASE_REF", "") + target_branch = target_branch.removeprefix("refs/heads/") + print(f"target branch: {target_branch!r}") + + head_branch = os.environ.get("GITHUB_HEAD_REF", "") + head_branch = head_branch.removeprefix("refs/heads/") + print(f"head branch: {head_branch!r}") + return target_branch, head_branch + + +def get_changed_files(ref_a: str = "main", ref_b: str = "HEAD") -> Set[Path]: + """List the files changed between two Git refs, filtered by change type.""" + args = ("git", "diff", "--name-only", f"{ref_a}...{ref_b}", "--") + print(*args) + changed_files_result = subprocess.run( + args, stdout=subprocess.PIPE, check=True, encoding="utf-8" + ) + changed_files = changed_files_result.stdout.strip().splitlines() + return frozenset(map(Path, filter(None, map(str.strip, changed_files)))) + + +def process_changed_files(changed_files: Set[Path]) -> Outputs: + run_tests = False + run_ci_fuzz = False + run_docs = False + run_windows_msi = False + + for file in changed_files: + file_name = file.name + file_suffix = file.suffix + file_parts = file.parts + + # Documentation files + doc_or_misc = file_parts[0] in {"Doc", "Misc"} + doc_file = file_suffix in SUFFIXES_DOCUMENTATION or doc_or_misc + + if file.parent == GITHUB_WORKFLOWS_PATH: + if file_name == "build.yml": + run_tests = run_ci_fuzz = True + if file_name == "reusable-docs.yml": + run_docs = True + if file_name == "reusable-windows-msi.yml": + run_windows_msi = True + + if not ( + doc_file + or file == GITHUB_CODEOWNERS_PATH + or file_name in CONFIGURATION_FILE_NAMES + ): + run_tests = True + + # The fuzz tests are pretty slow so they are executed only for PRs + # changing relevant files. + if file_suffix in SUFFIXES_C_OR_CPP: + run_ci_fuzz = True + if file_parts[:2] in { + ("configure",), + ("Modules", "_xxtestfuzz"), + }: + run_ci_fuzz = True + + # Check for changed documentation-related files + if doc_file: + run_docs = True + + # Check for changed MSI installer-related files + if file_parts[:2] == ("Tools", "msi"): + run_windows_msi = True + + return Outputs( + run_ci_fuzz=run_ci_fuzz, + run_docs=run_docs, + run_tests=run_tests, + run_windows_msi=run_windows_msi, + ) + + +def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs: + if not git_branch: + outputs.run_tests = True + + # Check if we should run the hypothesis tests + if git_branch in {"3.8", "3.9", "3.10", "3.11"}: + print("Branch too old for hypothesis tests") + outputs.run_hypothesis = False + else: + outputs.run_hypothesis = outputs.run_tests + + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so CIFuzz should be run only for code that is likely to be + # merged into the main branch; compatibility with older branches may + # be broken. + if git_branch != "main": + outputs.run_ci_fuzz = False + + if os.environ.get("GITHUB_EVENT_NAME", "").lower() == "workflow_dispatch": + outputs.run_docs = True + outputs.run_windows_msi = True + + return outputs + + +def write_github_output(outputs: Outputs) -> None: + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#default-environment-variables + # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/workflow-commands-for-github-actions#setting-an-output-parameter + if "GITHUB_OUTPUT" not in os.environ: + print("GITHUB_OUTPUT not defined!") + return + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: + f.write(f"run-cifuzz={bool_lower(outputs.run_ci_fuzz)}\n") + f.write(f"run-docs={bool_lower(outputs.run_docs)}\n") + f.write(f"run-hypothesis={bool_lower(outputs.run_hypothesis)}\n") + f.write(f"run-tests={bool_lower(outputs.run_tests)}\n") + f.write(f"run-win-msi={bool_lower(outputs.run_windows_msi)}\n") + + +def bool_lower(value: bool, /) -> str: + return "true" if value else "false" + + +if __name__ == "__main__": + compute_changes() From 4c0a7a4dd0694ac733daa772bef59a861b6eea0d Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Tue, 4 Feb 2025 10:55:44 +0000 Subject: [PATCH 02/14] Hugo's review Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-change-detection.yml | 2 +- Tools/build/compute-changes.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 5f43ead83617ee..c74b6467ffe45f 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -89,7 +89,7 @@ jobs: COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) - # Get all commits since that commit date from the base branch (eg: master or main): + # Get all commits since that commit date from the base branch (eg: main): git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules env: diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 42fbec7bc5283b..b51176facc83f0 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -25,8 +25,8 @@ ".ruff.toml", "mypy.ini", }) -SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"}) SUFFIXES_C_OR_CPP = frozenset({".c", ".h", ".cpp"}) +SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"}) @dataclass(kw_only=True, slots=True) @@ -38,7 +38,7 @@ class Outputs: run_windows_msi: bool = False -def compute_changes(): +def compute_changes() -> None: target_branch, head_branch = git_branches() if target_branch and head_branch: # Getting changed files only makes sense on a pull request @@ -55,7 +55,7 @@ def compute_changes(): print("Run tests") if outputs.run_hypothesis: - print("Run hypothesis tests") + print("Run Hypothesis tests") if outputs.run_ci_fuzz: print("Run CIFuzz tests") @@ -155,14 +155,14 @@ def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs: if not git_branch: outputs.run_tests = True - # Check if we should run the hypothesis tests - if git_branch in {"3.8", "3.9", "3.10", "3.11"}: - print("Branch too old for hypothesis tests") + # Check if we should run the Hypothesis tests + if git_branch in {"3.9", "3.10", "3.11"}: + print("Branch too old for Hypothesis tests") outputs.run_hypothesis = False else: outputs.run_hypothesis = outputs.run_tests - # oss-fuzz maintains a configuration for fuzzing the main branch of + # OSS-Fuzz maintains a configuration for fuzzing the main branch of # CPython, so CIFuzz should be run only for code that is likely to be # merged into the main branch; compatibility with older branches may # be broken. From 4169b462b0701d718c5f3e521454a12c42281934 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:04:00 +0000 Subject: [PATCH 03/14] Simplify --- Tools/build/compute-changes.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index b51176facc83f0..847b441afab50f 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -102,34 +102,30 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_windows_msi = False for file in changed_files: - file_name = file.name - file_suffix = file.suffix - file_parts = file.parts - # Documentation files - doc_or_misc = file_parts[0] in {"Doc", "Misc"} - doc_file = file_suffix in SUFFIXES_DOCUMENTATION or doc_or_misc + doc_or_misc = file.parts[0] in {"Doc", "Misc"} + doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc if file.parent == GITHUB_WORKFLOWS_PATH: - if file_name == "build.yml": + if file.name == "build.yml": run_tests = run_ci_fuzz = True - if file_name == "reusable-docs.yml": + if file.name == "reusable-docs.yml": run_docs = True - if file_name == "reusable-windows-msi.yml": + if file.name == "reusable-windows-msi.yml": run_windows_msi = True if not ( doc_file or file == GITHUB_CODEOWNERS_PATH - or file_name in CONFIGURATION_FILE_NAMES + or file.name in CONFIGURATION_FILE_NAMES ): run_tests = True # The fuzz tests are pretty slow so they are executed only for PRs # changing relevant files. - if file_suffix in SUFFIXES_C_OR_CPP: + if file.suffix in SUFFIXES_C_OR_CPP: run_ci_fuzz = True - if file_parts[:2] in { + if file.parts[:2] in { ("configure",), ("Modules", "_xxtestfuzz"), }: @@ -140,7 +136,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: run_docs = True # Check for changed MSI installer-related files - if file_parts[:2] == ("Tools", "msi"): + if file.parts[:2] == ("Tools", "msi"): run_windows_msi = True return Outputs( From 49f02be2a35085e1e3a867251e44688c18af2c84 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Tue, 4 Feb 2025 11:12:40 +0000 Subject: [PATCH 04/14] Standardise workflow variables: * Rename `config_hash` to `config-hash` * Rename `run_tests` to `run-tests` * Rename `run-win-msi` to `run-windows-msi` * Rename `run_hypothesis` to `run-hypothesis` * Rename `run_cifuzz` to `run-ci-fuzz` --- .github/workflows/build.yml | 54 +++++++++---------- .../workflows/reusable-change-detection.yml | 20 +++---- Tools/build/compute-changes.py | 4 +- 3 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c10c5b4aa46ffb..db28a5494acda7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,7 +30,7 @@ jobs: # if: fromJSON(needs.check_source.outputs.run-docs) # # ${{ - # fromJSON(needs.check_source.outputs.run_tests) + # fromJSON(needs.check_source.outputs.run-tests) # && 'truthy-branch' # || 'falsy-branch' # }} @@ -52,7 +52,7 @@ jobs: image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' steps: - name: Install Git run: | @@ -95,7 +95,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' steps: - uses: actions/checkout@v4 with: @@ -110,7 +110,7 @@ jobs: with: path: config.cache # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH @@ -154,7 +154,7 @@ jobs: Windows ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: fromJSON(needs.check_source.outputs.run_tests) + if: fromJSON(needs.check_source.outputs.run-tests) strategy: fail-fast: false matrix: @@ -185,7 +185,7 @@ jobs: name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category Windows MSI${{ '' }} needs: check_source - if: fromJSON(needs.check_source.outputs.run-win-msi) + if: fromJSON(needs.check_source.outputs.run-windows-msi) strategy: matrix: arch: @@ -201,7 +201,7 @@ jobs: macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' strategy: fail-fast: false matrix: @@ -226,7 +226,7 @@ jobs: free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + config_hash: ${{ needs.check_source.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -236,7 +236,7 @@ jobs: ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' strategy: matrix: bolt: @@ -257,7 +257,7 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + config_hash: ${{ needs.check_source.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -267,7 +267,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' strategy: fail-fast: false matrix: @@ -289,7 +289,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -327,17 +327,17 @@ jobs: build_wasi: name: 'WASI' needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' uses: ./.github/workflows/reusable-wasi.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + config_hash: ${{ needs.check_source.outputs.config-hash }} test_hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true' + if: needs.check_source.outputs.run-tests == 'true' && needs.check_source.outputs.run-hypothesis == 'true' env: OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 @@ -384,7 +384,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -453,7 +453,7 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' strategy: matrix: os: [ubuntu-24.04] @@ -471,7 +471,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -516,7 +516,7 @@ jobs: Thread sanitizer ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' strategy: matrix: free-threading: @@ -524,14 +524,14 @@ jobs: - true uses: ./.github/workflows/reusable-tsan.yml with: - config_hash: ${{ needs.check_source.outputs.config_hash }} + config_hash: ${{ needs.check_source.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest needs: check_source - if: needs.check_source.outputs.run_tests == 'true' + if: needs.check_source.outputs.run-tests == 'true' steps: - uses: actions/checkout@v4 with: @@ -542,7 +542,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir @@ -572,7 +572,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run_cifuzz == 'true' + if: needs.check_source.outputs.run-ci-fuzz == 'true' permissions: security-events: write strategy: @@ -611,7 +611,7 @@ jobs: if: always() needs: - - check_source # Transitive dependency, needed to access `run_tests` value + - check_source # Transitive dependency, needed to access `run-tests` value - check-docs - check_autoconf_regen - check_generated_files @@ -646,7 +646,7 @@ jobs: || '' }} ${{ - needs.check_source.outputs.run_tests != 'true' + needs.check_source.outputs.run-tests != 'true' && ' check_autoconf_regen, check_generated_files, @@ -661,14 +661,14 @@ jobs: || '' }} ${{ - !fromJSON(needs.check_source.outputs.run_cifuzz) + !fromJSON(needs.check_source.outputs.run-ci-fuzz) && ' cifuzz, ' || '' }} ${{ - !fromJSON(needs.check_source.outputs.run_hypothesis) + !fromJSON(needs.check_source.outputs.run-hypothesis) && ' test_hypothesis, ' diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index c74b6467ffe45f..554ac951c165c0 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -23,27 +23,27 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config_hash: + config-hash: description: Config hash value for use in cache keys value: ${{ jobs.compute-changes.outputs.config-hash }} # str run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool - run_tests: + run-tests: description: Whether to run the regular tests value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool - run-win-msi: + run-windows-msi: description: Whether to run the MSI installer smoke tests value: >- # bool - ${{ jobs.compute-changes.outputs.run-win-msi || false }} - run_hypothesis: + ${{ jobs.compute-changes.outputs.run-windows-msi || false }} # bool + run-hypothesis: description: Whether to run the Hypothesis tests value: >- # bool - ${{ jobs.compute-changes.outputs.run-hypothesis || false }} - run_cifuzz: + ${{ jobs.compute-changes.outputs.run-hypothesis || false }} # bool + run-ci-fuzz: description: Whether to run the CIFuzz job value: >- # bool - ${{ jobs.compute-changes.outputs.run-cifuzz || false }} + ${{ jobs.compute-changes.outputs.run-ci-fuzz || false }} # bool jobs: compute-changes: @@ -52,11 +52,11 @@ jobs: timeout-minutes: 10 outputs: config-hash: ${{ steps.config-hash.outputs.hash }} - run-cifuzz: ${{ steps.changes.outputs.run-cifuzz }} + run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} run-docs: ${{ steps.changes.outputs.run-docs }} run-hypothesis: ${{ steps.changes.outputs.run-hypothesis }} run-tests: ${{ steps.changes.outputs.run-tests }} - run-win-msi: ${{ steps.changes.outputs.run-win-msi }} + run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} steps: - uses: actions/setup-python@v5 with: diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 847b441afab50f..4d7abeea8804fd 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -180,11 +180,11 @@ def write_github_output(outputs: Outputs) -> None: return with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: - f.write(f"run-cifuzz={bool_lower(outputs.run_ci_fuzz)}\n") + f.write(f"run-ci-fuzz={bool_lower(outputs.run_ci_fuzz)}\n") f.write(f"run-docs={bool_lower(outputs.run_docs)}\n") f.write(f"run-hypothesis={bool_lower(outputs.run_hypothesis)}\n") f.write(f"run-tests={bool_lower(outputs.run_tests)}\n") - f.write(f"run-win-msi={bool_lower(outputs.run_windows_msi)}\n") + f.write(f"run-windows-msi={bool_lower(outputs.run_windows_msi)}\n") def bool_lower(value: bool, /) -> str: From d65e5be2b1cb6902ea3d6f70adcc509694a680da Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 07:53:00 +0000 Subject: [PATCH 05/14] no # bool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- .github/workflows/reusable-change-detection.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 554ac951c165c0..88a9188dc58f76 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -35,15 +35,15 @@ on: # yamllint disable-line rule:truthy run-windows-msi: description: Whether to run the MSI installer smoke tests value: >- # bool - ${{ jobs.compute-changes.outputs.run-windows-msi || false }} # bool + ${{ jobs.compute-changes.outputs.run-windows-msi || false }} run-hypothesis: description: Whether to run the Hypothesis tests value: >- # bool - ${{ jobs.compute-changes.outputs.run-hypothesis || false }} # bool + ${{ jobs.compute-changes.outputs.run-hypothesis || false }} run-ci-fuzz: description: Whether to run the CIFuzz job value: >- # bool - ${{ jobs.compute-changes.outputs.run-ci-fuzz || false }} # bool + ${{ jobs.compute-changes.outputs.run-ci-fuzz || false }} jobs: compute-changes: From b40113e52f3066756fd6b39188f6aae3e6632df4 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 14:43:31 +0000 Subject: [PATCH 06/14] Add step name --- .github/workflows/reusable-change-detection.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 88a9188dc58f76..fa4185c62b31d4 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -58,7 +58,8 @@ jobs: run-tests: ${{ steps.changes.outputs.run-tests }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} steps: - - uses: actions/setup-python@v5 + - name: Set up Python + uses: actions/setup-python@v5 with: python-version: "3" From 1ee4e0e4de7b26bb718ef301b3e771ae95619f9a Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 14:48:06 +0000 Subject: [PATCH 07/14] empty From e183b1521a715faeb1e7516c3a58b178090931b9 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:15:39 +0000 Subject: [PATCH 08/14] Merge run_hypothesis into run_tests --- .github/workflows/build.yml | 10 ++-------- .github/workflows/reusable-change-detection.yml | 5 ----- Tools/build/compute-changes.py | 12 ------------ 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index db28a5494acda7..5de748f5870aaa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -337,7 +337,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 60 needs: check_source - if: needs.check_source.outputs.run-tests == 'true' && needs.check_source.outputs.run-hypothesis == 'true' + if: needs.check_source.outputs.run-tests == 'true' env: OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 @@ -657,6 +657,7 @@ jobs: build_windows, build_asan, build_tsan, + test_hypothesis, ' || '' }} @@ -667,11 +668,4 @@ jobs: ' || '' }} - ${{ - !fromJSON(needs.check_source.outputs.run-hypothesis) - && ' - test_hypothesis, - ' - || '' - }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index fa4185c62b31d4..57475a454b5b9b 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -36,10 +36,6 @@ on: # yamllint disable-line rule:truthy description: Whether to run the MSI installer smoke tests value: >- # bool ${{ jobs.compute-changes.outputs.run-windows-msi || false }} - run-hypothesis: - description: Whether to run the Hypothesis tests - value: >- # bool - ${{ jobs.compute-changes.outputs.run-hypothesis || false }} run-ci-fuzz: description: Whether to run the CIFuzz job value: >- # bool @@ -54,7 +50,6 @@ jobs: config-hash: ${{ steps.config-hash.outputs.hash }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} run-docs: ${{ steps.changes.outputs.run-docs }} - run-hypothesis: ${{ steps.changes.outputs.run-hypothesis }} run-tests: ${{ steps.changes.outputs.run-tests }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} steps: diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 4d7abeea8804fd..b0816553556b8a 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -33,7 +33,6 @@ class Outputs: run_ci_fuzz: bool = False run_docs: bool = False - run_hypothesis: bool = False run_tests: bool = False run_windows_msi: bool = False @@ -54,9 +53,6 @@ def compute_changes() -> None: if outputs.run_tests: print("Run tests") - if outputs.run_hypothesis: - print("Run Hypothesis tests") - if outputs.run_ci_fuzz: print("Run CIFuzz tests") else: @@ -151,13 +147,6 @@ def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs: if not git_branch: outputs.run_tests = True - # Check if we should run the Hypothesis tests - if git_branch in {"3.9", "3.10", "3.11"}: - print("Branch too old for Hypothesis tests") - outputs.run_hypothesis = False - else: - outputs.run_hypothesis = outputs.run_tests - # OSS-Fuzz maintains a configuration for fuzzing the main branch of # CPython, so CIFuzz should be run only for code that is likely to be # merged into the main branch; compatibility with older branches may @@ -182,7 +171,6 @@ def write_github_output(outputs: Outputs) -> None: with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: f.write(f"run-ci-fuzz={bool_lower(outputs.run_ci_fuzz)}\n") f.write(f"run-docs={bool_lower(outputs.run_docs)}\n") - f.write(f"run-hypothesis={bool_lower(outputs.run_hypothesis)}\n") f.write(f"run-tests={bool_lower(outputs.run_tests)}\n") f.write(f"run-windows-msi={bool_lower(outputs.run_windows_msi)}\n") From e03d2959c1ebc84c07b36e67f1a1623045fe4b70 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:22:51 +0000 Subject: [PATCH 09/14] Use github.event.repository.default_branch per review --- .github/workflows/reusable-change-detection.yml | 2 ++ Tools/build/compute-changes.py | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 57475a454b5b9b..409a4a2f615e90 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -99,6 +99,8 @@ jobs: - name: Compute changed files id: changes run: python Tools/build/compute-changes.py + env: + GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - name: Compute hash for config cache key id: config-hash diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index b0816553556b8a..629223a8b09c66 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -18,6 +18,7 @@ if TYPE_CHECKING: from collections.abc import Set +GITHUB_DEFAULT_BRANCH = os.environ.get("GITHUB_DEFAULT_BRANCH", "") GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") GITHUB_WORKFLOWS_PATH = Path(".github/workflows") CONFIGURATION_FILE_NAMES = frozenset({ @@ -80,7 +81,9 @@ def git_branches() -> tuple[str, str]: return target_branch, head_branch -def get_changed_files(ref_a: str = "main", ref_b: str = "HEAD") -> Set[Path]: +def get_changed_files( + ref_a: str = GITHUB_DEFAULT_BRANCH, ref_b: str = "HEAD" +) -> Set[Path]: """List the files changed between two Git refs, filtered by change type.""" args = ("git", "diff", "--name-only", f"{ref_a}...{ref_b}", "--") print(*args) @@ -147,11 +150,8 @@ def process_target_branch(outputs: Outputs, git_branch: str) -> Outputs: if not git_branch: outputs.run_tests = True - # OSS-Fuzz maintains a configuration for fuzzing the main branch of - # CPython, so CIFuzz should be run only for code that is likely to be - # merged into the main branch; compatibility with older branches may - # be broken. - if git_branch != "main": + # CIFuzz / OSS-Fuzz compatibility with older branches may be broken. + if git_branch != GITHUB_DEFAULT_BRANCH: outputs.run_ci_fuzz = False if os.environ.get("GITHUB_EVENT_NAME", "").lower() == "workflow_dispatch": From 05253e32b8f6322d6b46da2678c2736d9e654f52 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:26:07 +0000 Subject: [PATCH 10/14] Update Tools/build/compute-changes.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) --- Tools/build/compute-changes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 629223a8b09c66..0bd10d72db7f42 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from collections.abc import Set -GITHUB_DEFAULT_BRANCH = os.environ.get("GITHUB_DEFAULT_BRANCH", "") +GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH] GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") GITHUB_WORKFLOWS_PATH = Path(".github/workflows") CONFIGURATION_FILE_NAMES = frozenset({ From f45c009327d3dad6668c64b682eb72166a31c1cb Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:27:47 +0000 Subject: [PATCH 11/14] fix syntax error in suggestion --- Tools/build/compute-changes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 0bd10d72db7f42..162f6fd6eeb636 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -18,7 +18,7 @@ if TYPE_CHECKING: from collections.abc import Set -GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH] +GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"] GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") GITHUB_WORKFLOWS_PATH = Path(".github/workflows") CONFIGURATION_FILE_NAMES = frozenset({ From e265fa5fadb9946ca017327989fec1af5db7dfb8 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:54:21 +0000 Subject: [PATCH 12/14] Rename to "context" from "change-detection" --- .github/workflows/build.yml | 88 +++++++++---------- ...nge-detection.yml => reusable-context.yml} | 34 +++---- Tools/build/compute-changes.py | 2 +- 3 files changed, 58 insertions(+), 66 deletions(-) rename .github/workflows/{reusable-change-detection.yml => reusable-context.yml} (70%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5de748f5870aaa..b58e80f69933c5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,15 +22,15 @@ env: FORCE_COLOR: 1 jobs: - check_source: + build-context: name: Change detection # To use boolean outputs from this job, parse them as JSON. # Here's some examples: # - # if: fromJSON(needs.check_source.outputs.run-docs) + # if: fromJSON(needs.build-context.outputs.run-docs) # # ${{ - # fromJSON(needs.check_source.outputs.run-tests) + # fromJSON(needs.build-context.outputs.run-tests) # && 'truthy-branch' # || 'falsy-branch' # }} @@ -39,8 +39,8 @@ jobs: check-docs: name: Docs - needs: check_source - if: fromJSON(needs.check_source.outputs.run-docs) + needs: build-context + if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml check_autoconf_regen: @@ -51,8 +51,8 @@ jobs: container: image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' steps: - name: Install Git run: | @@ -94,8 +94,8 @@ jobs: # reproducible: to get the same tools versions (autoconf, aclocal, ...) runs-on: ubuntu-24.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' steps: - uses: actions/checkout@v4 with: @@ -110,7 +110,7 @@ jobs: with: path: config.cache # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }}-${{ env.pythonLocation }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Add ccache to PATH @@ -153,8 +153,8 @@ jobs: name: >- Windows ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: check_source - if: fromJSON(needs.check_source.outputs.run-tests) + needs: build-context + if: fromJSON(needs.build-context.outputs.run-tests) strategy: fail-fast: false matrix: @@ -184,8 +184,8 @@ jobs: build_windows_msi: name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category Windows MSI${{ '' }} - needs: check_source - if: fromJSON(needs.check_source.outputs.run-windows-msi) + needs: build-context + if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: matrix: arch: @@ -200,8 +200,8 @@ jobs: name: >- macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' strategy: fail-fast: false matrix: @@ -226,7 +226,7 @@ jobs: free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.check_source.outputs.config-hash }} + config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -235,8 +235,8 @@ jobs: Ubuntu ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' strategy: matrix: bolt: @@ -257,7 +257,7 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.check_source.outputs.config-hash }} + config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -266,8 +266,8 @@ jobs: name: 'Ubuntu SSL tests with OpenSSL' runs-on: ${{ matrix.os }} timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' strategy: fail-fast: false matrix: @@ -289,7 +289,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -326,18 +326,18 @@ jobs: build_wasi: name: 'WASI' - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' uses: ./.github/workflows/reusable-wasi.yml with: - config_hash: ${{ needs.check_source.outputs.config-hash }} + config_hash: ${{ needs.build-context.outputs.config-hash }} test_hypothesis: name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-24.04 timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' env: OPENSSL_VER: 3.0.15 PYTHONSTRICTEXTENSIONBUILD: 1 @@ -384,7 +384,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -452,8 +452,8 @@ jobs: name: 'Address sanitizer' runs-on: ${{ matrix.os }} timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' strategy: matrix: os: [ubuntu-24.04] @@ -471,7 +471,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -515,8 +515,8 @@ jobs: name: >- Thread sanitizer ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' strategy: matrix: free-threading: @@ -524,14 +524,14 @@ jobs: - true uses: ./.github/workflows/reusable-tsan.yml with: - config_hash: ${{ needs.check_source.outputs.config-hash }} + config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} cross-build-linux: name: Cross build Linux runs-on: ubuntu-latest - needs: check_source - if: needs.check_source.outputs.run-tests == 'true' + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' steps: - uses: actions/checkout@v4 with: @@ -542,7 +542,7 @@ jobs: uses: actions/cache@v4 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.check_source.outputs.config-hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir @@ -571,8 +571,8 @@ jobs: name: CIFuzz runs-on: ubuntu-latest timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run-ci-fuzz == 'true' + needs: build-context + if: needs.build-context.outputs.run-ci-fuzz == 'true' permissions: security-events: write strategy: @@ -611,7 +611,7 @@ jobs: if: always() needs: - - check_source # Transitive dependency, needed to access `run-tests` value + - build-context # Transitive dependency, needed to access `run-tests` value - check-docs - check_autoconf_regen - check_generated_files @@ -639,14 +639,14 @@ jobs: test_hypothesis, allowed-skips: >- ${{ - !fromJSON(needs.check_source.outputs.run-docs) + !fromJSON(needs.build-context.outputs.run-docs) && ' check-docs, ' || '' }} ${{ - needs.check_source.outputs.run-tests != 'true' + needs.build-context.outputs.run-tests != 'true' && ' check_autoconf_regen, check_generated_files, @@ -662,7 +662,7 @@ jobs: || '' }} ${{ - !fromJSON(needs.check_source.outputs.run-ci-fuzz) + !fromJSON(needs.build-context.outputs.run-ci-fuzz) && ' cifuzz, ' diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-context.yml similarity index 70% rename from .github/workflows/reusable-change-detection.yml rename to .github/workflows/reusable-context.yml index 409a4a2f615e90..f8d1aee7f1a01f 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-context.yml @@ -1,24 +1,18 @@ -name: Reusable change detection +name: Reusable build context on: # yamllint disable-line rule:truthy workflow_call: outputs: - # Some of the referenced steps set outputs conditionally and there may be - # cases when referencing them evaluates to empty strings. It is nice to - # work with proper booleans so they have to be evaluated through JSON - # conversion in the expressions. However, empty strings used like that - # may trigger all sorts of undefined and hard-to-debug behaviors in - # GitHub Actions CI/CD. To help with this, all of the outputs set here - # that are meant to be used as boolean flags (and not arbitrary strings), - # MUST have fallbacks with default values set. A common pattern would be - # to add ` || false` to all such expressions here, in the output - # definitions. They can then later be safely used through the following - # idiom in job conditionals and other expressions. Here's some examples: + # Every referenced step MUST always its set output variable, + # either via ``Tools/build/compute-changes.py`` or in this workflow file. + # Boolean outputs (generally prefixed ``run-``) can then later be used + # safely through the following idiom in job conditionals and other + # expressions. Here's some examples: # - # if: fromJSON(needs.change-detection.outputs.run-docs) + # if: fromJSON(needs.build-context.outputs.run-tests) # # ${{ - # fromJSON(needs.change-detection.outputs.run-tests) + # fromJSON(needs.build-context.outputs.run-tests) # && 'truthy-branch' # || 'falsy-branch' # }} @@ -28,22 +22,20 @@ on: # yamllint disable-line rule:truthy value: ${{ jobs.compute-changes.outputs.config-hash }} # str run-docs: description: Whether to build the docs - value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool + value: ${{ jobs.compute-changes.outputs.run-docs }} # bool run-tests: description: Whether to run the regular tests - value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool + value: ${{ jobs.compute-changes.outputs.run-tests }} # bool run-windows-msi: description: Whether to run the MSI installer smoke tests - value: >- # bool - ${{ jobs.compute-changes.outputs.run-windows-msi || false }} + value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool run-ci-fuzz: description: Whether to run the CIFuzz job - value: >- # bool - ${{ jobs.compute-changes.outputs.run-ci-fuzz || false }} + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool jobs: compute-changes: - name: Compute changed files + name: Create context from changed files runs-on: ubuntu-latest timeout-minutes: 10 outputs: diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index 162f6fd6eeb636..105ba58cc9d941 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -1,6 +1,6 @@ """Determine which GitHub Actions workflows to run. -Called by ``.github/workflows/reusable-change-detection.yml``. +Called by ``.github/workflows/reusable-context.yml``. We only want to run tests on PRs when related files are changed, or when someone triggers a manual workflow run. This improves developer experience by not doing (slow) From 06ef1172b2b19ff1295132ca41a5d4f53bbb7d05 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 15:59:33 +0000 Subject: [PATCH 13/14] fixup! Rename to "context" from "change-detection" --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b58e80f69933c5..dc2f4858be6e8c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: # || 'falsy-branch' # }} # - uses: ./.github/workflows/reusable-change-detection.yml + uses: ./.github/workflows/reusable-context.yml check-docs: name: Docs From f52c88f163c06d8e82e8870f72072a906cd5bc7b Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Wed, 5 Feb 2025 16:16:08 +0000 Subject: [PATCH 14/14] fixup! Rename to "context" from "change-detection" --- .github/workflows/reusable-context.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index f8d1aee7f1a01f..fa4df6f29711db 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -3,7 +3,7 @@ name: Reusable build context on: # yamllint disable-line rule:truthy workflow_call: outputs: - # Every referenced step MUST always its set output variable, + # Every referenced step MUST always set its output variable, # either via ``Tools/build/compute-changes.py`` or in this workflow file. # Boolean outputs (generally prefixed ``run-``) can then later be used # safely through the following idiom in job conditionals and other