From 6b4bd9eb79160a754ed788955836310129e8d286 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 23 May 2023 17:58:17 +0300 Subject: [PATCH 1/4] CI: Cache config.cache across runs to speed up build --- .github/workflows/build.yml | 51 ++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 41abddffa5d648..de20213a3de9f5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -81,6 +81,11 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v3 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} - uses: actions/setup-python@v3 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh @@ -97,7 +102,7 @@ jobs: - name: Configure CPython run: | # Build Python with the libpython dynamic library - ./configure --with-pydebug --enable-shared + ./configure --config-cache --with-pydebug --enable-shared - name: Regenerate autoconf files with container image run: make regen-configure - name: Build CPython @@ -178,6 +183,11 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v3 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk - name: Configure CPython @@ -186,6 +196,7 @@ jobs: LDFLAGS="-L$(brew --prefix gdbm)/lib -I$(brew --prefix xz)/lib" \ PKG_CONFIG_PATH="$(brew --prefix tcl-tk)/lib/pkgconfig" \ ./configure \ + --config-cache \ --with-pydebug \ --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@1.1)" @@ -238,9 +249,18 @@ jobs: run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -271,6 +291,11 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v3 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -295,7 +320,7 @@ jobs: - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython run: make -j4 - name: Display build info @@ -304,7 +329,7 @@ jobs: run: ./python Lib/test/ssltests.py test_hypothesis: - name: "Hypothesis Tests on Ubuntu" + name: "Hypothesis tests on Ubuntu" runs-on: ubuntu-20.04 timeout-minutes: 60 needs: check_source @@ -345,9 +370,18 @@ jobs: run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - name: Bind mount sources read-only run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: ${{ env.CPYTHON_BUILDDIR }}/config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR + run: | + ../cpython-ro-srcdir/configure \ + --config-cache \ + --with-pydebug \ + --with-openssl=$OPENSSL_DIR - name: Build CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j4 @@ -415,6 +449,11 @@ jobs: ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v3 + - name: Restore config.cache + uses: actions/cache@v3 + with: + path: config.cache + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -443,7 +482,7 @@ jobs: - name: Configure ccache action uses: hendrikmuhs/ccache-action@v1.2 - name: Configure CPython - run: ./configure --with-address-sanitizer --without-pymalloc + run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython run: make -j4 - name: Display build info From 7ef178c99b304147dbdf165b2497287cee2f9f1f Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 23 May 2023 20:17:35 +0300 Subject: [PATCH 2/4] Combine hashFiles --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de20213a3de9f5..57978bcbd50eb2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} + key: ${{ github.job }}-${{ hashFiles('configure', 'configure.ac') }} - uses: actions/setup-python@v3 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh @@ -187,7 +187,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk - name: Configure CPython @@ -253,7 +253,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -295,7 +295,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -374,7 +374,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -453,7 +453,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure') }}-${{ hashFiles('configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies From 539a1a096f2d7491ac1b0b478e06d162c2447fdc Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 23 May 2023 20:20:59 +0300 Subject: [PATCH 3/4] Add missing runner.os --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 57978bcbd50eb2..a1de1078155d39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} - uses: actions/setup-python@v3 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh From 5962941902a3b1446dbe715c3aa59a921feb6bbc Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 23 May 2023 20:55:27 +0300 Subject: [PATCH 4/4] Include workflow file in key, to invalidate when env vars and OS versions changed --- .github/workflows/build.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1de1078155d39..51296fc78e6a99 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,7 +85,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }} - uses: actions/setup-python@v3 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh @@ -187,7 +187,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }} - name: Install Homebrew dependencies run: brew install pkg-config openssl@1.1 xz gdbm tcl-tk - name: Configure CPython @@ -253,7 +253,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -295,7 +295,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -374,7 +374,7 @@ jobs: uses: actions/cache@v3 with: path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -453,7 +453,7 @@ jobs: uses: actions/cache@v3 with: path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac') }} + key: ${{ github.job }}-${{ runner.os }}-${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies