From 00b8559975e843d570081abd3d00f246ef788831 Mon Sep 17 00:00:00 2001 From: Noa Date: Tue, 25 Feb 2025 20:36:26 -0600 Subject: [PATCH] Switch to libz-rs-sys for zlib implementation --- .github/workflows/ci.yaml | 2 +- .github/workflows/cron-ci.yaml | 4 +-- .github/workflows/release.yml | 2 +- Cargo.lock | 24 ++++++++------ Cargo.toml | 3 +- stdlib/Cargo.toml | 5 ++- stdlib/src/zlib.rs | 57 +++++----------------------------- 7 files changed, 29 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e96c2f26a6..23f47b574c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl # Skip additional tests on Windows. They are checked on Linux and MacOS. # test_glob: many failing tests # test_io: many failing tests diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index bfc758310e..4b8e701ced 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -6,7 +6,7 @@ on: name: Periodic checks/tasks env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,ssl,jit PYTHON_VERSION: "3.13.1" jobs: @@ -24,7 +24,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - run: sudo apt-get update && sudo apt-get -y install lcov - name: Run cargo-llvm-cov with Rust tests. - run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,zlib,importlib,encodings,ssl,jit + run: cargo llvm-cov --no-report --workspace --exclude rustpython_wasm --verbose --no-default-features --features stdlib,importlib,encodings,ssl,jit - name: Run cargo-llvm-cov with Python snippets. run: python scripts/cargo-llvm-cov.py continue-on-error: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5554ed0a8b..99035d04fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,7 +16,7 @@ permissions: contents: write env: - CARGO_ARGS: --no-default-features --features stdlib,zlib,importlib,encodings,sqlite,ssl + CARGO_ARGS: --no-default-features --features stdlib,importlib,encodings,sqlite,ssl jobs: build: diff --git a/Cargo.lock b/Cargo.lock index 9d486786ba..67e0813b7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -720,12 +720,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", - "libz-sys", + "libz-rs-sys", "miniz_oxide", ] @@ -1137,14 +1137,12 @@ dependencies = [ ] [[package]] -name = "libz-sys" -version = "1.1.21" +name = "libz-rs-sys" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +checksum = "902bc563b5d65ad9bba616b490842ef0651066a1a1dc3ce1087113ffcb873c8d" dependencies = [ - "cc", - "pkg-config", - "vcpkg", + "zlib-rs", ] [[package]] @@ -2210,7 +2208,7 @@ dependencies = [ "junction", "libc", "libsqlite3-sys", - "libz-sys", + "libz-rs-sys", "mac_address", "malachite-bigint", "md-5", @@ -3479,3 +3477,9 @@ dependencies = [ "quote", "syn 2.0.98", ] + +[[package]] +name = "zlib-rs" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b20717f0917c908dc63de2e44e97f1e6b126ca58d0e391cee86d504eb8fbd05" diff --git a/Cargo.toml b/Cargo.toml index a649a7ed0f..6c48714929 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "zlib", "importlib"] +default = ["threading", "stdlib", "importlib"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdlib = ["rustpython-stdlib", "rustpython-pylib", "encodings"] @@ -18,7 +18,6 @@ flame-it = ["rustpython-vm/flame-it", "flame", "flamescope"] freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/freeze-stdlib"] jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] -zlib = ["stdlib", "rustpython-stdlib/zlib"] bz2 = ["stdlib", "rustpython-stdlib/bz2"] sqlite = ["rustpython-stdlib/sqlite"] ssl = ["rustpython-stdlib/ssl"] diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index 37e17e2fc8..1086dea090 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -14,7 +14,6 @@ license.workspace = true default = ["compiler"] compiler = ["rustpython-vm/compiler"] threading = ["rustpython-common/threading", "rustpython-vm/threading"] -zlib = ["libz-sys", "flate2/zlib"] bz2 = ["bzip2"] sqlite = ["dep:libsqlite3-sys"] ssl = ["openssl", "openssl-sys", "foreign-types-shared", "openssl-probe"] @@ -48,7 +47,6 @@ memchr = { workspace = true } base64 = "0.13.0" csv-core = "0.1.11" dyn-clone = "1.0.10" -libz-sys = { version = "1.1", default-features = false, optional = true } puruspe = "0.4.0" xml-rs = "0.8.14" @@ -81,7 +79,8 @@ ucd = "0.1.1" # compression adler32 = "1.2.0" crc32fast = "1.3.2" -flate2 = "1.0.28" +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } +libz-sys = { package = "libz-rs-sys", version = "0.4" } bzip2 = { version = "0.4", optional = true } # uuid diff --git a/stdlib/src/zlib.rs b/stdlib/src/zlib.rs index 83a7535c33..a4828552a7 100644 --- a/stdlib/src/zlib.rs +++ b/stdlib/src/zlib.rs @@ -9,6 +9,7 @@ mod zlib { common::lock::PyMutex, convert::TryFromBorrowedObject, function::{ArgBytesLike, ArgPrimitiveIndex, ArgSize, OptionalArg}, + types::Constructor, PyObject, PyPayload, PyResult, VirtualMachine, }; use adler32::RollingAdler32 as Adler32; @@ -19,35 +20,12 @@ mod zlib { }; use std::io::Write; - #[cfg(not(feature = "zlib"))] - mod constants { - pub const Z_NO_COMPRESSION: i32 = 0; - pub const Z_BEST_COMPRESSION: i32 = 9; - pub const Z_BEST_SPEED: i32 = 1; - pub const Z_DEFAULT_COMPRESSION: i32 = -1; - pub const Z_NO_FLUSH: i32 = 0; - pub const Z_PARTIAL_FLUSH: i32 = 1; - pub const Z_SYNC_FLUSH: i32 = 2; - pub const Z_FULL_FLUSH: i32 = 3; - // not sure what the value here means, but it's the only compression method zlibmodule - // supports, so it doesn't really matter - pub const Z_DEFLATED: i32 = 8; - } - #[cfg(feature = "zlib")] - use libz_sys as constants; - - #[pyattr] - use constants::{ - Z_BEST_COMPRESSION, Z_BEST_SPEED, Z_DEFAULT_COMPRESSION, Z_DEFLATED as DEFLATED, - Z_FULL_FLUSH, Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_SYNC_FLUSH, - }; - - #[cfg(feature = "zlib")] #[pyattr] use libz_sys::{ - Z_BLOCK, Z_DEFAULT_STRATEGY, Z_FILTERED, Z_FINISH, Z_FIXED, Z_HUFFMAN_ONLY, Z_RLE, Z_TREES, + Z_BEST_COMPRESSION, Z_BEST_SPEED, Z_BLOCK, Z_DEFAULT_COMPRESSION, Z_DEFAULT_STRATEGY, + Z_DEFLATED as DEFLATED, Z_FILTERED, Z_FINISH, Z_FIXED, Z_FULL_FLUSH, Z_HUFFMAN_ONLY, + Z_NO_COMPRESSION, Z_NO_FLUSH, Z_PARTIAL_FLUSH, Z_RLE, Z_SYNC_FLUSH, Z_TREES, }; - use rustpython_vm::types::Constructor; // copied from zlibmodule.c (commit 530f506ac91338) #[pyattr] @@ -119,11 +97,11 @@ mod zlib { header: bool, // [De]Compress::new_with_window_bits is only enabled for zlib; miniz_oxide doesn't // support wbits (yet?) - #[cfg(feature = "zlib")] wbits: u8, }, - #[cfg(feature = "zlib")] - Gzip { wbits: u8 }, + Gzip { + wbits: u8, + }, } impl InitOptions { @@ -131,12 +109,7 @@ mod zlib { let header = wbits > 0; let wbits = wbits.unsigned_abs(); match wbits { - 9..=15 => Ok(InitOptions::Standard { - header, - #[cfg(feature = "zlib")] - wbits, - }), - #[cfg(feature = "zlib")] + 9..=15 => Ok(InitOptions::Standard { header, wbits }), 25..=31 => Ok(InitOptions::Gzip { wbits: wbits - 16 }), _ => Err(vm.new_value_error("Invalid initialization option".to_owned())), } @@ -144,23 +117,15 @@ mod zlib { fn decompress(self) -> Decompress { match self { - #[cfg(not(feature = "zlib"))] - Self::Standard { header } => Decompress::new(header), - #[cfg(feature = "zlib")] Self::Standard { header, wbits } => Decompress::new_with_window_bits(header, wbits), - #[cfg(feature = "zlib")] Self::Gzip { wbits } => Decompress::new_gzip(wbits), } } fn compress(self, level: Compression) -> Compress { match self { - #[cfg(not(feature = "zlib"))] - Self::Standard { header } => Compress::new(level, header), - #[cfg(feature = "zlib")] Self::Standard { header, wbits } => { Compress::new_with_window_bits(level, header, wbits) } - #[cfg(feature = "zlib")] Self::Gzip { wbits } => Compress::new_gzip(level, wbits), } } @@ -264,7 +229,6 @@ mod zlib { struct DecompressobjArgs { #[pyarg(any, default = "ArgPrimitiveIndex { value: MAX_WBITS }")] wbits: ArgPrimitiveIndex, - #[cfg(feature = "zlib")] #[pyarg(any, optional)] _zdict: OptionalArg, } @@ -273,7 +237,6 @@ mod zlib { fn decompressobj(args: DecompressobjArgs, vm: &VirtualMachine) -> PyResult { #[allow(unused_mut)] let mut decompress = InitOptions::new(args.wbits.value, vm)?.decompress(); - #[cfg(feature = "zlib")] if let OptionalArg::Present(_dict) = args._zdict { // FIXME: always fails // dict.with_ref(|d| decompress.set_dictionary(d)); @@ -426,10 +389,8 @@ mod zlib { wbits: ArgPrimitiveIndex, #[pyarg(any, name = "_memLevel", default = "DEF_MEM_LEVEL")] _mem_level: u8, - #[cfg(feature = "zlib")] #[pyarg(any, default = "Z_DEFAULT_STRATEGY")] _strategy: i32, - #[cfg(feature = "zlib")] #[pyarg(any, optional)] zdict: Option, } @@ -439,7 +400,6 @@ mod zlib { let CompressobjArgs { level, wbits, - #[cfg(feature = "zlib")] zdict, .. } = args; @@ -447,7 +407,6 @@ mod zlib { level.ok_or_else(|| vm.new_value_error("invalid initialization option".to_owned()))?; #[allow(unused_mut)] let mut compress = InitOptions::new(wbits.value, vm)?.compress(level); - #[cfg(feature = "zlib")] if let Some(zdict) = zdict { zdict.with_ref(|zdict| compress.set_dictionary(zdict).unwrap()); }