From e7f25ca9eca1d297601382a46371a56e24b73999 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 23 Aug 2022 15:56:35 -0500 Subject: [PATCH 1/7] Only run lalrpop generation on ubuntu-latest --- .github/workflows/ci.yaml | 11 ++--------- .github/workflows/cron-ci.yaml | 11 ++--------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5b2877083b..77f0207765 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -285,10 +285,7 @@ jobs: lalrpop: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Generate parser with lalrpop - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Cache generated parser @@ -301,13 +298,9 @@ jobs: uses: andstor/file-existence-action@v1 with: files: "compiler/parser/python.rs" - - if: runner.os == 'Windows' - name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes - run: | - set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file - name: Install lalrpop if: steps.generated_parser.outputs.files_exists == 'false' - uses: baptiste0928/cargo-install@v1 + uses: actions-rs/install@v0.1 with: crate: lalrpop version: "0.19.8" diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 453f4b8f64..d028186ac8 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -186,10 +186,7 @@ jobs: lalrpop: name: Generate parser with lalrpop - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Cache generated parser @@ -202,13 +199,9 @@ jobs: uses: andstor/file-existence-action@v1 with: files: "compiler/parser/python.rs" - - if: runner.os == 'Windows' - name: Force python.lalrpop to be lf # actions@checkout ignore .gitattributes - run: | - set file compiler/parser/python.lalrpop; ((Get-Content $file) -join "`n") + "`n" | Set-Content -NoNewline $file - name: Install lalrpop if: steps.generated_parser.outputs.files_exists == 'false' - uses: baptiste0928/cargo-install@v1 + uses: actions-rs/install@v0.1 with: crate: lalrpop version: "0.19.8" From f3f32c6f0b81e23066a643a0a99953e9c53e6854 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 24 Aug 2022 17:59:20 -0500 Subject: [PATCH 2/7] Checkout with lf on windows --- .github/workflows/ci.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 77f0207765..3cd8cb117c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -111,6 +111,8 @@ jobs: os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false steps: + # for windows, always checkout files with their original lf line endings + - run: git config --global core.autocrlf false; git config --global core.eol lf - uses: actions/checkout@v2 - name: Cache generated parser uses: actions/cache@v2 @@ -227,6 +229,7 @@ jobs: os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false steps: + - run: git config --global core.autocrlf false; git config --global core.eol lf - uses: actions/checkout@v2 - name: Cache generated parser uses: actions/cache@v2 From 626e30f20d74dfeb39a68d94e232ff5e83180011 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 24 Aug 2022 19:07:50 -0500 Subject: [PATCH 3/7] Use actions/cache@v3 --- .github/workflows/ci.yaml | 16 ++++++++-------- .github/workflows/cron-ci.yaml | 10 +++++----- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 3cd8cb117c..0774244823 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -115,7 +115,7 @@ jobs: - run: git config --global core.autocrlf false; git config --global core.eol lf - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -160,7 +160,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -232,7 +232,7 @@ jobs: - run: git config --global core.autocrlf false; git config --global core.eol lf - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -292,7 +292,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -318,7 +318,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -356,7 +356,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -378,7 +378,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -427,7 +427,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index d028186ac8..ecf54fa5f3 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -67,7 +67,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -102,7 +102,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -140,7 +140,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} @@ -190,7 +190,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Cache generated parser - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: compiler/parser/python.rs key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} From d3fc78952c1ea773e6d7d798db1f89e677c98de7 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 24 Aug 2022 19:36:50 -0500 Subject: [PATCH 4/7] New hash for python.lalrpop --- compiler/parser/python.lalrpop | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/parser/python.lalrpop b/compiler/parser/python.lalrpop index 224450e073..44d988206d 100644 --- a/compiler/parser/python.lalrpop +++ b/compiler/parser/python.lalrpop @@ -51,7 +51,7 @@ SimpleStatement: ast::Suite = { let mut statements = vec![s1]; statements.extend(s2.into_iter().map(|e| e.1)); statements - } + }, }; SmallStatement: ast::Stmt = { @@ -138,7 +138,7 @@ ExpressionStatement: ast::Stmt = { }; AssignSuffix: ast::Expr = { - "=" => e + "=" => e, }; TestListOrYieldExpr: ast::Expr = { From 3b00204a32ca3542b60a45efecc910dbb60b81d9 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 24 Aug 2022 20:10:26 -0500 Subject: [PATCH 5/7] Cache lalrpop binary, not generated file --- .github/workflows/ci.yaml | 90 ++++++++-------------------------- .github/workflows/cron-ci.yaml | 61 +++++------------------ 2 files changed, 33 insertions(+), 118 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 0774244823..239094a0fb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -104,7 +104,6 @@ jobs: env: RUST_BACKTRACE: full name: Run rust tests - needs: lalrpop runs-on: ${{ matrix.os }} strategy: matrix: @@ -114,11 +113,9 @@ jobs: # for windows, always checkout files with their original lf line endings - run: git config --global core.autocrlf false; git config --global core.eol lf - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable - name: Set up the Windows environment shell: bash @@ -155,15 +152,12 @@ jobs: exotic_targets: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Ensure compilation on various targets - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable with: @@ -219,7 +213,6 @@ jobs: snippets_cpython: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - needs: lalrpop env: RUST_BACKTRACE: full name: Run snippets and cpython tests @@ -231,11 +224,9 @@ jobs: steps: - run: git config --global core.autocrlf false; git config --global core.eol lf - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v2 @@ -284,44 +275,14 @@ jobs: run: | mkdir site-packages target/release/rustpython --install-pip ensurepip --user - - lalrpop: - if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} - name: Generate parser with lalrpop - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - - name: Check if cached generated parser exists - id: generated_parser - uses: andstor/file-existence-action@v1 - with: - files: "compiler/parser/python.rs" - - name: Install lalrpop - if: steps.generated_parser.outputs.files_exists == 'false' - uses: actions-rs/install@v0.1 - with: - crate: lalrpop - version: "0.19.8" - - name: Run lalrpop - if: steps.generated_parser.outputs.files_exists == 'false' - run: lalrpop compiler/parser/python.lalrpop - lint: name: Check Rust code with rustfmt and clippy - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable with: components: rustfmt, clippy @@ -351,15 +312,12 @@ jobs: miri: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run tests under miri - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@master with: toolchain: nightly @@ -373,15 +331,12 @@ jobs: wasm: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Check the WASM package and demo - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v1 - name: install wasm-pack @@ -422,15 +377,12 @@ jobs: wasm-wasi: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run snippets and cpython tests on wasm-wasi - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasi diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index ecf54fa5f3..5c6aa97e6d 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -11,15 +11,12 @@ env: jobs: codecov: name: Collect code coverage data - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable with: components: llvm-tools-preview @@ -62,15 +59,12 @@ jobs: testdata: name: Collect regression test data - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable - name: build rustpython run: cargo build --release --verbose @@ -97,15 +91,12 @@ jobs: whatsleft: name: Collect what is left data - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable - name: build rustpython run: cargo build --release --verbose @@ -135,15 +126,12 @@ jobs: benchmark: name: Collect benchmark data - needs: lalrpop runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} + - uses: actions/cache@v3 + with: { path: "~/.cargo/bin/lalrpop\n~/.cargo/.crates.toml", key: "lalrpop-${{ runner.os }}" } + - run: cargo install lalrpop@0.19.8 - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v2 with: @@ -183,28 +171,3 @@ jobs: if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update benchmark results"; then git push fi - - lalrpop: - name: Generate parser with lalrpop - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Cache generated parser - uses: actions/cache@v3 - with: - path: compiler/parser/python.rs - key: lalrpop-${{ hashFiles('compiler/parser/python.lalrpop') }} - - name: Check if cached generated parser exists - id: generated_parser - uses: andstor/file-existence-action@v1 - with: - files: "compiler/parser/python.rs" - - name: Install lalrpop - if: steps.generated_parser.outputs.files_exists == 'false' - uses: actions-rs/install@v0.1 - with: - crate: lalrpop - version: "0.19.8" - - name: Run lalrpop - if: steps.generated_parser.outputs.files_exists == 'false' - run: lalrpop compiler/parser/python.lalrpop From 4bf507d159c7811db191d70167be9211990ec176 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 24 Aug 2022 21:00:38 -0500 Subject: [PATCH 6/7] Run lalrpop from build.rs --- compiler/parser/build.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/compiler/parser/build.rs b/compiler/parser/build.rs index 81c28f43be..7483dc084b 100644 --- a/compiler/parser/build.rs +++ b/compiler/parser/build.rs @@ -78,16 +78,30 @@ fn try_lalrpop(source: &str, target: &str) -> anyhow::Result<()> { }; #[cfg(feature = "lalrpop")] - lalrpop::process_root().unwrap_or_else(|e| { - println!("cargo:warning={_message}"); - panic!("running lalrpop failed. {e:?}"); - }); + { + lalrpop::process_root().unwrap_or_else(|e| { + println!("cargo:warning={_message}"); + panic!("running lalrpop failed. {e:?}"); + }); + Ok(()) + } #[cfg(not(feature = "lalrpop"))] { - println!("cargo:warning=try: cargo build --manifest-path=compiler/parser/Cargo.toml --features=lalrpop"); + match std::process::Command::new("lalrpop").arg(source).status() { + Ok(status) => { + if status.success() { + Ok(()) + } else { + Err(anyhow::anyhow!("running lalrpop failed: {status}")) + } + } + Err(_) => panic!( + "lalrpop not installed. try: cargo build \ + --manifest-path=compiler/parser/Cargo.toml --features=lalrpop" + ), + } } - Ok(()) } fn sha_equal(expected_sha3_str: &str, actual_sha3: &[u8; 32]) -> bool { From 1dfefa675ad9c789db8e47f32003ced86cdc4585 Mon Sep 17 00:00:00 2001 From: Noa Date: Wed, 24 Aug 2022 22:25:43 -0500 Subject: [PATCH 7/7] Unmark some expected failures now that windows has lf --- Lib/test/test_difflib.py | 4 ---- Lib/test/test_imp.py | 4 ---- 2 files changed, 8 deletions(-) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 68da83dda2..f3dce104b8 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -244,10 +244,6 @@ def test_html_diff(self): with open(findfile('test_difflib_expect.html')) as fp: self.assertEqual(actual, fp.read()) - # TODO: RUSTPYTHON - if sys.platform == "win32": - test_html_diff = unittest.expectedFailure(test_html_diff) - def test_recursion_limit(self): # Check if the problem described in patch #1413711 exists. limit = sys.getrecursionlimit() diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index f06a120d3c..c1a9d6f43c 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -100,10 +100,6 @@ def test_issue1267(self): self.assertEqual(fp.readline(), '"""Tokenization help for Python programs.\n') - # TODO: RUSTPYTHON - if sys.platform == 'win32': - test_issue1267 = unittest.expectedFailure(test_issue1267) - def test_issue3594(self): temp_mod_name = 'test_imp_helper' sys.path.insert(0, '.')